youtube_dl/extractor/youtube.py

   1 # coding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5 import itertools
   6 import json
   7 import os.path
   8 import random
   9 import re
  10 import traceback
  11
  12 from .common import InfoExtractor, SearchInfoExtractor
  13 from ..compat import (
  14     compat_chr,
  15     compat_HTTPError,
  16     compat_parse_qs,
  17     compat_str,
  18     compat_urllib_parse_unquote_plus,
  19     compat_urllib_parse_urlencode,
  20     compat_urllib_parse_urlparse,
  21     compat_urlparse,
  22 )
  23 from ..jsinterp import JSInterpreter
  24 from ..utils import (
  25     ExtractorError,
  26     clean_html,
  27     dict_get,
  28     float_or_none,
  29     int_or_none,
  30     mimetype2ext,
  31     parse_codecs,
  32     parse_duration,
  33     qualities,
  34     remove_start,
  35     smuggle_url,
  36     str_or_none,
  37     str_to_int,
  38     try_get,
  39     unescapeHTML,
  40     unified_strdate,
  41     unsmuggle_url,
  42     update_url_query,
  43     url_or_none,
  44     urlencode_postdata,
  45     urljoin,
  46 )
  47
  48
  49 def parse_qs(url):
  50     return compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
  51
  52
  53 class YoutubeBaseInfoExtractor(InfoExtractor):
  54     """Provide base functions for Youtube extractors"""
  55     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
  56     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
  57
  58     _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
  59     _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
  60     _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
  61
  62     _NETRC_MACHINE = 'youtube'
  63     # If True it will raise an error if no login info is provided
  64     _LOGIN_REQUIRED = False
  65
  66     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)'
  67
  68     def _ids_to_results(self, ids):
  69         return [
  70             self.url_result(vid_id, 'Youtube', video_id=vid_id)
  71             for vid_id in ids]
  72
  73     def _login(self):
  74         """
  75         Attempt to log in to YouTube.
  76         True is returned if successful or skipped.
  77         False is returned if login failed.
  78
  79         If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
  80         """
  81         username, password = self._get_login_info()
  82         # No authentication to be performed
  83         if username is None:
  84             if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None:
  85                 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
  86             return True
  87
  88         login_page = self._download_webpage(
  89             self._LOGIN_URL, None,
  90             note='Downloading login page',
  91             errnote='unable to fetch login page', fatal=False)
  92         if login_page is False:
  93             return
  94
  95         login_form = self._hidden_inputs(login_page)
  96
  97         def req(url, f_req, note, errnote):
  98             data = login_form.copy()
  99             data.update({
 100                 'pstMsg': 1,
 101                 'checkConnection': 'youtube',
 102                 'checkedDomains': 'youtube',
 103                 'hl': 'en',
 104                 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
 105                 'f.req': json.dumps(f_req),
 106                 'flowName': 'GlifWebSignIn',
 107                 'flowEntry': 'ServiceLogin',
 108                 # TODO: reverse actual botguard identifier generation algo
 109                 'bgRequest': '["identifier",""]',
 110             })
 111             return self._download_json(
 112                 url, None, note=note, errnote=errnote,
 113                 transform_source=lambda s: re.sub(r'^[^[]*', '', s),
 114                 fatal=False,
 115                 data=urlencode_postdata(data), headers={
 116                     'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
 117                     'Google-Accounts-XSRF': 1,
 118                 })
 119
 120         def warn(message):
 121             self._downloader.report_warning(message)
 122
 123         lookup_req = [
 124             username,
 125             None, [], None, 'US', None, None, 2, False, True,
 126             [
 127                 None, None,
 128                 [2, 1, None, 1,
 129                  'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
 130                  None, [], 4],
 131                 1, [None, None, []], None, None, None, True
 132             ],
 133             username,
 134         ]
 135
 136         lookup_results = req(
 137             self._LOOKUP_URL, lookup_req,
 138             'Looking up account info', 'Unable to look up account info')
 139
 140         if lookup_results is False:
 141             return False
 142
 143         user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
 144         if not user_hash:
 145             warn('Unable to extract user hash')
 146             return False
 147
 148         challenge_req = [
 149             user_hash,
 150             None, 1, None, [1, None, None, None, [password, None, True]],
 151             [
 152                 None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
 153                 1, [None, None, []], None, None, None, True
 154             ]]
 155
 156         challenge_results = req(
 157             self._CHALLENGE_URL, challenge_req,
 158             'Logging in', 'Unable to log in')
 159
 160         if challenge_results is False:
 161             return
 162
 163         login_res = try_get(challenge_results, lambda x: x[0][5], list)
 164         if login_res:
 165             login_msg = try_get(login_res, lambda x: x[5], compat_str)
 166             warn(
 167                 'Unable to login: %s' % 'Invalid password'
 168                 if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
 169             return False
 170
 171         res = try_get(challenge_results, lambda x: x[0][-1], list)
 172         if not res:
 173             warn('Unable to extract result entry')
 174             return False
 175
 176         login_challenge = try_get(res, lambda x: x[0][0], list)
 177         if login_challenge:
 178             challenge_str = try_get(login_challenge, lambda x: x[2], compat_str)
 179             if challenge_str == 'TWO_STEP_VERIFICATION':
 180                 # SEND_SUCCESS - TFA code has been successfully sent to phone
 181                 # QUOTA_EXCEEDED - reached the limit of TFA codes
 182                 status = try_get(login_challenge, lambda x: x[5], compat_str)
 183                 if status == 'QUOTA_EXCEEDED':
 184                     warn('Exceeded the limit of TFA codes, try later')
 185                     return False
 186
 187                 tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
 188                 if not tl:
 189                     warn('Unable to extract TL')
 190                     return False
 191
 192                 tfa_code = self._get_tfa_info('2-step verification code')
 193
 194                 if not tfa_code:
 195                     warn(
 196                         'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
 197                         '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
 198                     return False
 199
 200                 tfa_code = remove_start(tfa_code, 'G-')
 201
 202                 tfa_req = [
 203                     user_hash, None, 2, None,
 204                     [
 205                         9, None, None, None, None, None, None, None,
 206                         [None, tfa_code, True, 2]
 207                     ]]
 208
 209                 tfa_results = req(
 210                     self._TFA_URL.format(tl), tfa_req,
 211                     'Submitting TFA code', 'Unable to submit TFA code')
 212
 213                 if tfa_results is False:
 214                     return False
 215
 216                 tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
 217                 if tfa_res:
 218                     tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
 219                     warn(
 220                         'Unable to finish TFA: %s' % 'Invalid TFA code'
 221                         if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
 222                     return False
 223
 224                 check_cookie_url = try_get(
 225                     tfa_results, lambda x: x[0][-1][2], compat_str)
 226             else:
 227                 CHALLENGES = {
 228                     'LOGIN_CHALLENGE': "This device isn't recognized. For your security, Google wants to make sure it's really you.",
 229                     'USERNAME_RECOVERY': 'Please provide additional information to aid in the recovery process.',
 230                     'REAUTH': "There is something unusual about your activity. For your security, Google wants to make sure it's really you.",
 231                 }
 232                 challenge = CHALLENGES.get(
 233                     challenge_str,
 234                     '%s returned error %s.' % (self.IE_NAME, challenge_str))
 235                 warn('%s\nGo to https://accounts.google.com/, login and solve a challenge.' % challenge)
 236                 return False
 237         else:
 238             check_cookie_url = try_get(res, lambda x: x[2], compat_str)
 239
 240         if not check_cookie_url:
 241             warn('Unable to extract CheckCookie URL')
 242             return False
 243
 244         check_cookie_results = self._download_webpage(
 245             check_cookie_url, None, 'Checking cookie', fatal=False)
 246
 247         if check_cookie_results is False:
 248             return False
 249
 250         if 'https://myaccount.google.com/' not in check_cookie_results:
 251             warn('Unable to log in')
 252             return False
 253
 254         return True
 255
 256     def _initialize_consent(self):
 257         cookies = self._get_cookies('https://www.youtube.com/')
 258         if cookies.get('__Secure-3PSID'):
 259             return
 260         consent_id = None
 261         consent = cookies.get('CONSENT')
 262         if consent:
 263             if 'YES' in consent.value:
 264                 return
 265             consent_id = self._search_regex(
 266                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 267         if not consent_id:
 268             consent_id = random.randint(100, 999)
 269         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 270
 271     def _real_initialize(self):
 272         self._initialize_consent()
 273         if self._downloader is None:
 274             return
 275         if not self._login():
 276             return
 277
 278     _DEFAULT_API_DATA = {
 279         'context': {
 280             'client': {
 281                 'clientName': 'WEB',
 282                 'clientVersion': '2.20201021.03.00',
 283             }
 284         },
 285     }
 286
 287     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
 288     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
 289     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 290
 291     def _call_api(self, ep, query, video_id, fatal=True):
 292         data = self._DEFAULT_API_DATA.copy()
 293         data.update(query)
 294
 295         return self._download_json(
 296             'https://www.youtube.com/youtubei/v1/%s' % ep, video_id=video_id,
 297             note='Downloading API JSON', errnote='Unable to download API page',
 298             data=json.dumps(data).encode('utf8'), fatal=fatal,
 299             headers={'content-type': 'application/json'},
 300             query={'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'})
 301
 302     def _extract_yt_initial_data(self, video_id, webpage):
 303         return self._parse_json(
 304             self._search_regex(
 305                 (r'%s\s*%s' % (self._YT_INITIAL_DATA_RE, self._YT_INITIAL_BOUNDARY_RE),
 306                  self._YT_INITIAL_DATA_RE), webpage, 'yt initial data'),
 307             video_id)
 308
 309     def _extract_ytcfg(self, video_id, webpage):
 310         return self._parse_json(
 311             self._search_regex(
 312                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 313                 default='{}'), video_id, fatal=False) or {}
 314
 315     def _extract_video(self, renderer):
 316         video_id = renderer['videoId']
 317         title = try_get(
 318             renderer,
 319             (lambda x: x['title']['runs'][0]['text'],
 320              lambda x: x['title']['simpleText']), compat_str)
 321         description = try_get(
 322             renderer, lambda x: x['descriptionSnippet']['runs'][0]['text'],
 323             compat_str)
 324         duration = parse_duration(try_get(
 325             renderer, lambda x: x['lengthText']['simpleText'], compat_str))
 326         view_count_text = try_get(
 327             renderer, lambda x: x['viewCountText']['simpleText'], compat_str) or ''
 328         view_count = str_to_int(self._search_regex(
 329             r'^([\d,]+)', re.sub(r'\s', '', view_count_text),
 330             'view count', default=None))
 331         uploader = try_get(
 332             renderer,
 333             (lambda x: x['ownerText']['runs'][0]['text'],
 334              lambda x: x['shortBylineText']['runs'][0]['text']), compat_str)
 335         return {
 336             '_type': 'url',
 337             'ie_key': YoutubeIE.ie_key(),
 338             'id': video_id,
 339             'url': video_id,
 340             'title': title,
 341             'description': description,
 342             'duration': duration,
 343             'view_count': view_count,
 344             'uploader': uploader,
 345         }
 346
 347
 348 class YoutubeIE(YoutubeBaseInfoExtractor):
 349     IE_DESC = 'YouTube.com'
 350     _INVIDIOUS_SITES = (
 351         # invidious-redirect websites
 352         r'(?:www\.)?redirect\.invidious\.io',
 353         r'(?:(?:www|dev)\.)?invidio\.us',
 354         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
 355         r'(?:(?:www|no)\.)?invidiou\.sh',
 356         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 357         r'(?:www\.)?invidious\.kabi\.tk',
 358         r'(?:www\.)?invidious\.13ad\.de',
 359         r'(?:www\.)?invidious\.mastodon\.host',
 360         r'(?:www\.)?invidious\.zapashcanon\.fr',
 361         r'(?:www\.)?invidious\.kavin\.rocks',
 362         r'(?:www\.)?invidious\.tinfoil-hat\.net',
 363         r'(?:www\.)?invidious\.himiko\.cloud',
 364         r'(?:www\.)?invidious\.reallyancient\.tech',
 365         r'(?:www\.)?invidious\.tube',
 366         r'(?:www\.)?invidiou\.site',
 367         r'(?:www\.)?invidious\.site',
 368         r'(?:www\.)?invidious\.xyz',
 369         r'(?:www\.)?invidious\.nixnet\.xyz',
 370         r'(?:www\.)?invidious\.048596\.xyz',
 371         r'(?:www\.)?invidious\.drycat\.fr',
 372         r'(?:www\.)?inv\.skyn3t\.in',
 373         r'(?:www\.)?tube\.poal\.co',
 374         r'(?:www\.)?tube\.connect\.cafe',
 375         r'(?:www\.)?vid\.wxzm\.sx',
 376         r'(?:www\.)?vid\.mint\.lgbt',
 377         r'(?:www\.)?vid\.puffyan\.us',
 378         r'(?:www\.)?yewtu\.be',
 379         r'(?:www\.)?yt\.elukerio\.org',
 380         r'(?:www\.)?yt\.lelux\.fi',
 381         r'(?:www\.)?invidious\.ggc-project\.de',
 382         r'(?:www\.)?yt\.maisputain\.ovh',
 383         r'(?:www\.)?ytprivate\.com',
 384         r'(?:www\.)?invidious\.13ad\.de',
 385         r'(?:www\.)?invidious\.toot\.koeln',
 386         r'(?:www\.)?invidious\.fdn\.fr',
 387         r'(?:www\.)?watch\.nettohikari\.com',
 388         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 389         r'(?:www\.)?qklhadlycap4cnod\.onion',
 390         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 391         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 392         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 393         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 394         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 395         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 396     )
 397     _VALID_URL = r"""(?x)^
 398                      (
 399                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 400                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 401                             (?:www\.)?deturl\.com/www\.youtube\.com|
 402                             (?:www\.)?pwnyoutube\.com|
 403                             (?:www\.)?hooktube\.com|
 404                             (?:www\.)?yourepeat\.com|
 405                             tube\.majestyc\.net|
 406                             %(invidious)s|
 407                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 408                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 409                          (?:                                                  # the various things that can precede the ID:
 410                              (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
 411                              |(?:                                             # or the v= param in all its forms
 412                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 413                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 414                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 415                                  v=
 416                              )
 417                          ))
 418                          |(?:
 419                             youtu\.be|                                        # just youtu.be/xxxx
 420                             vid\.plus|                                        # or vid.plus/xxxx
 421                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 422                             %(invidious)s
 423                          )/
 424                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 425                          )
 426                      )?                                                       # all until now is optional -> you can pass the naked ID
 427                      (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
 428                      (?(1).+)?                                                # if we found the ID, everything can follow
 429                      $""" % {
 430         'invidious': '|'.join(_INVIDIOUS_SITES),
 431     }
 432     _PLAYER_INFO_RE = (
 433         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 434         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 435         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 436     )
 437     _SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt')
 438
 439     _GEO_BYPASS = False
 440
 441     IE_NAME = 'youtube'
 442     _TESTS = [
 443         {
 444             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
 445             'info_dict': {
 446                 'id': 'BaW_jenozKc',
 447                 'ext': 'mp4',
 448                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
 449                 'uploader': 'Philipp Hagemeister',
 450                 'uploader_id': 'phihag',
 451                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
 452                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
 453                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
 454                 'upload_date': '20121002',
 455                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
 456                 'categories': ['Science & Technology'],
 457                 'tags': ['youtube-dl'],
 458                 'duration': 10,
 459                 'view_count': int,
 460                 'like_count': int,
 461                 'dislike_count': int,
 462                 'start_time': 1,
 463                 'end_time': 9,
 464             }
 465         },
 466         {
 467             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
 468             'note': 'Embed-only video (#1746)',
 469             'info_dict': {
 470                 'id': 'yZIXLfi8CZQ',
 471                 'ext': 'mp4',
 472                 'upload_date': '20120608',
 473                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
 474                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
 475                 'uploader': 'SET India',
 476                 'uploader_id': 'setindia',
 477                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
 478                 'age_limit': 18,
 479             },
 480             'skip': 'Private video',
 481         },
 482         {
 483             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
 484             'note': 'Use the first video ID in the URL',
 485             'info_dict': {
 486                 'id': 'BaW_jenozKc',
 487                 'ext': 'mp4',
 488                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
 489                 'uploader': 'Philipp Hagemeister',
 490                 'uploader_id': 'phihag',
 491                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
 492                 'upload_date': '20121002',
 493                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
 494                 'categories': ['Science & Technology'],
 495                 'tags': ['youtube-dl'],
 496                 'duration': 10,
 497                 'view_count': int,
 498                 'like_count': int,
 499                 'dislike_count': int,
 500             },
 501             'params': {
 502                 'skip_download': True,
 503             },
 504         },
 505         {
 506             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
 507             'note': '256k DASH audio (format 141) via DASH manifest',
 508             'info_dict': {
 509                 'id': 'a9LDPn-MO4I',
 510                 'ext': 'm4a',
 511                 'upload_date': '20121002',
 512                 'uploader_id': '8KVIDEO',
 513                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
 514                 'description': '',
 515                 'uploader': '8KVIDEO',
 516                 'title': 'UHDTV TEST 8K VIDEO.mp4'
 517             },
 518             'params': {
 519                 'youtube_include_dash_manifest': True,
 520                 'format': '141',
 521             },
 522             'skip': 'format 141 not served anymore',
 523         },
 524         # DASH manifest with encrypted signature
 525         {
 526             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
 527             'info_dict': {
 528                 'id': 'IB3lcPjvWLA',
 529                 'ext': 'm4a',
 530                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
 531                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
 532                 'duration': 244,
 533                 'uploader': 'AfrojackVEVO',
 534                 'uploader_id': 'AfrojackVEVO',
 535                 'upload_date': '20131011',
 536                 'abr': 129.495,
 537             },
 538             'params': {
 539                 'youtube_include_dash_manifest': True,
 540                 'format': '141/bestaudio[ext=m4a]',
 541             },
 542         },
 543         # Controversy video
 544         {
 545             'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
 546             'info_dict': {
 547                 'id': 'T4XJQO3qol8',
 548                 'ext': 'mp4',
 549                 'duration': 219,
 550                 'upload_date': '20100909',
 551                 'uploader': 'Amazing Atheist',
 552                 'uploader_id': 'TheAmazingAtheist',
 553                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
 554                 'title': 'Burning Everyone\'s Koran',
 555                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms \r\n\r\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
 556             }
 557         },
 558         # Normal age-gate video (No vevo, embed allowed), available via embed page
 559         {
 560             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
 561             'info_dict': {
 562                 'id': 'HtVdAasjOgU',
 563                 'ext': 'mp4',
 564                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
 565                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
 566                 'duration': 142,
 567                 'uploader': 'The Witcher',
 568                 'uploader_id': 'WitcherGame',
 569                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
 570                 'upload_date': '20140605',
 571                 'age_limit': 18,
 572             },
 573         },
 574         {
 575             # Age-gated video only available with authentication (unavailable
 576             # via embed page workaround)
 577             'url': 'XgnwCQzjau8',
 578             'only_matching': True,
 579         },
 580         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
 581         # YouTube Red ad is not captured for creator
 582         {
 583             'url': '__2ABJjxzNo',
 584             'info_dict': {
 585                 'id': '__2ABJjxzNo',
 586                 'ext': 'mp4',
 587                 'duration': 266,
 588                 'upload_date': '20100430',
 589                 'uploader_id': 'deadmau5',
 590                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
 591                 'creator': 'deadmau5',
 592                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
 593                 'uploader': 'deadmau5',
 594                 'title': 'Deadmau5 - Some Chords (HD)',
 595                 'alt_title': 'Some Chords',
 596             },
 597             'expected_warnings': [
 598                 'DASH manifest missing',
 599             ]
 600         },
 601         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
 602         {
 603             'url': 'lqQg6PlCWgI',
 604             'info_dict': {
 605                 'id': 'lqQg6PlCWgI',
 606                 'ext': 'mp4',
 607                 'duration': 6085,
 608                 'upload_date': '20150827',
 609                 'uploader_id': 'olympic',
 610                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
 611                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
 612                 'uploader': 'Olympic',
 613                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
 614             },
 615             'params': {
 616                 'skip_download': 'requires avconv',
 617             }
 618         },
 619         # Non-square pixels
 620         {
 621             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
 622             'info_dict': {
 623                 'id': '_b-2C3KPAM0',
 624                 'ext': 'mp4',
 625                 'stretched_ratio': 16 / 9.,
 626                 'duration': 85,
 627                 'upload_date': '20110310',
 628                 'uploader_id': 'AllenMeow',
 629                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
 630                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
 631                 'uploader': '孫ᄋᄅ',
 632                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
 633             },
 634         },
 635         # url_encoded_fmt_stream_map is empty string
 636         {
 637             'url': 'qEJwOuvDf7I',
 638             'info_dict': {
 639                 'id': 'qEJwOuvDf7I',
 640                 'ext': 'webm',
 641                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
 642                 'description': '',
 643                 'upload_date': '20150404',
 644                 'uploader_id': 'spbelect',
 645                 'uploader': 'Наблюдатели Петербурга',
 646             },
 647             'params': {
 648                 'skip_download': 'requires avconv',
 649             },
 650             'skip': 'This live event has ended.',
 651         },
 652         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
 653         {
 654             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
 655             'info_dict': {
 656                 'id': 'FIl7x6_3R5Y',
 657                 'ext': 'webm',
 658                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
 659                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
 660                 'duration': 220,
 661                 'upload_date': '20150625',
 662                 'uploader_id': 'dorappi2000',
 663                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
 664                 'uploader': 'dorappi2000',
 665                 'formats': 'mincount:31',
 666             },
 667             'skip': 'not actual anymore',
 668         },
 669         # DASH manifest with segment_list
 670         {
 671             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
 672             'md5': '8ce563a1d667b599d21064e982ab9e31',
 673             'info_dict': {
 674                 'id': 'CsmdDsKjzN8',
 675                 'ext': 'mp4',
 676                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
 677                 'uploader': 'Airtek',
 678                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
 679                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
 680                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
 681             },
 682             'params': {
 683                 'youtube_include_dash_manifest': True,
 684                 'format': '135',  # bestvideo
 685             },
 686             'skip': 'This live event has ended.',
 687         },
 688         {
 689             # Multifeed videos (multiple cameras), URL is for Main Camera
 690             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
 691             'info_dict': {
 692                 'id': 'jvGDaLqkpTg',
 693                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
 694                 'description': 'md5:e03b909557865076822aa169218d6a5d',
 695             },
 696             'playlist': [{
 697                 'info_dict': {
 698                     'id': 'jvGDaLqkpTg',
 699                     'ext': 'mp4',
 700                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
 701                     'description': 'md5:e03b909557865076822aa169218d6a5d',
 702                     'duration': 10643,
 703                     'upload_date': '20161111',
 704                     'uploader': 'Team PGP',
 705                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
 706                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
 707                 },
 708             }, {
 709                 'info_dict': {
 710                     'id': '3AKt1R1aDnw',
 711                     'ext': 'mp4',
 712                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
 713                     'description': 'md5:e03b909557865076822aa169218d6a5d',
 714                     'duration': 10991,
 715                     'upload_date': '20161111',
 716                     'uploader': 'Team PGP',
 717                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
 718                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
 719                 },
 720             }, {
 721                 'info_dict': {
 722                     'id': 'RtAMM00gpVc',
 723                     'ext': 'mp4',
 724                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
 725                     'description': 'md5:e03b909557865076822aa169218d6a5d',
 726                     'duration': 10995,
 727                     'upload_date': '20161111',
 728                     'uploader': 'Team PGP',
 729                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
 730                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
 731                 },
 732             }, {
 733                 'info_dict': {
 734                     'id': '6N2fdlP3C5U',
 735                     'ext': 'mp4',
 736                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
 737                     'description': 'md5:e03b909557865076822aa169218d6a5d',
 738                     'duration': 10990,
 739                     'upload_date': '20161111',
 740                     'uploader': 'Team PGP',
 741                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
 742                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
 743                 },
 744             }],
 745             'params': {
 746                 'skip_download': True,
 747             },
 748         },
 749         {
 750             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
 751             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
 752             'info_dict': {
 753                 'id': 'gVfLd0zydlo',
 754                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
 755             },
 756             'playlist_count': 2,
 757             'skip': 'Not multifeed anymore',
 758         },
 759         {
 760             'url': 'https://vid.plus/FlRa-iH7PGw',
 761             'only_matching': True,
 762         },
 763         {
 764             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
 765             'only_matching': True,
 766         },
 767         {
 768             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
 769             # Also tests cut-off URL expansion in video description (see
 770             # https://github.com/ytdl-org/youtube-dl/issues/1892,
 771             # https://github.com/ytdl-org/youtube-dl/issues/8164)
 772             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
 773             'info_dict': {
 774                 'id': 'lsguqyKfVQg',
 775                 'ext': 'mp4',
 776                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
 777                 'alt_title': 'Dark Walk - Position Music',
 778                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
 779                 'duration': 133,
 780                 'upload_date': '20151119',
 781                 'uploader_id': 'IronSoulElf',
 782                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
 783                 'uploader': 'IronSoulElf',
 784                 'creator': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
 785                 'track': 'Dark Walk - Position Music',
 786                 'artist': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
 787                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
 788             },
 789             'params': {
 790                 'skip_download': True,
 791             },
 792         },
 793         {
 794             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
 795             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
 796             'only_matching': True,
 797         },
 798         {
 799             # Video with yt:stretch=17:0
 800             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
 801             'info_dict': {
 802                 'id': 'Q39EVAstoRM',
 803                 'ext': 'mp4',
 804                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
 805                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
 806                 'upload_date': '20151107',
 807                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
 808                 'uploader': 'CH GAMER DROID',
 809             },
 810             'params': {
 811                 'skip_download': True,
 812             },
 813             'skip': 'This video does not exist.',
 814         },
 815         {
 816             # Video with incomplete 'yt:stretch=16:'
 817             'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
 818             'only_matching': True,
 819         },
 820         {
 821             # Video licensed under Creative Commons
 822             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
 823             'info_dict': {
 824                 'id': 'M4gD1WSo5mA',
 825                 'ext': 'mp4',
 826                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
 827                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
 828                 'duration': 721,
 829                 'upload_date': '20150127',
 830                 'uploader_id': 'BerkmanCenter',
 831                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
 832                 'uploader': 'The Berkman Klein Center for Internet & Society',
 833                 'license': 'Creative Commons Attribution license (reuse allowed)',
 834             },
 835             'params': {
 836                 'skip_download': True,
 837             },
 838         },
 839         {
 840             # Channel-like uploader_url
 841             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
 842             'info_dict': {
 843                 'id': 'eQcmzGIKrzg',
 844                 'ext': 'mp4',
 845                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
 846                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
 847                 'duration': 4060,
 848                 'upload_date': '20151119',
 849                 'uploader': 'Bernie Sanders',
 850                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
 851                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
 852                 'license': 'Creative Commons Attribution license (reuse allowed)',
 853             },
 854             'params': {
 855                 'skip_download': True,
 856             },
 857         },
 858         {
 859             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
 860             'only_matching': True,
 861         },
 862         {
 863             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
 864             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
 865             'only_matching': True,
 866         },
 867         {
 868             # Rental video preview
 869             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
 870             'info_dict': {
 871                 'id': 'uGpuVWrhIzE',
 872                 'ext': 'mp4',
 873                 'title': 'Piku - Trailer',
 874                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
 875                 'upload_date': '20150811',
 876                 'uploader': 'FlixMatrix',
 877                 'uploader_id': 'FlixMatrixKaravan',
 878                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
 879                 'license': 'Standard YouTube License',
 880             },
 881             'params': {
 882                 'skip_download': True,
 883             },
 884             'skip': 'This video is not available.',
 885         },
 886         {
 887             # YouTube Red video with episode data
 888             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
 889             'info_dict': {
 890                 'id': 'iqKdEhx-dD4',
 891                 'ext': 'mp4',
 892                 'title': 'Isolation - Mind Field (Ep 1)',
 893                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
 894                 'duration': 2085,
 895                 'upload_date': '20170118',
 896                 'uploader': 'Vsauce',
 897                 'uploader_id': 'Vsauce',
 898                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
 899                 'series': 'Mind Field',
 900                 'season_number': 1,
 901                 'episode_number': 1,
 902             },
 903             'params': {
 904                 'skip_download': True,
 905             },
 906             'expected_warnings': [
 907                 'Skipping DASH manifest',
 908             ],
 909         },
 910         {
 911             # The following content has been identified by the YouTube community
 912             # as inappropriate or offensive to some audiences.
 913             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
 914             'info_dict': {
 915                 'id': '6SJNVb0GnPI',
 916                 'ext': 'mp4',
 917                 'title': 'Race Differences in Intelligence',
 918                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
 919                 'duration': 965,
 920                 'upload_date': '20140124',
 921                 'uploader': 'New Century Foundation',
 922                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
 923                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
 924             },
 925             'params': {
 926                 'skip_download': True,
 927             },
 928             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
 929         },
 930         {
 931             # itag 212
 932             'url': '1t24XAntNCY',
 933             'only_matching': True,
 934         },
 935         {
 936             # geo restricted to JP
 937             'url': 'sJL6WA-aGkQ',
 938             'only_matching': True,
 939         },
 940         {
 941             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
 942             'only_matching': True,
 943         },
 944         {
 945             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
 946             'only_matching': True,
 947         },
 948         {
 949             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
 950             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
 951             'only_matching': True,
 952         },
 953         {
 954             # DRM protected
 955             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
 956             'only_matching': True,
 957         },
 958         {
 959             # Video with unsupported adaptive stream type formats
 960             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
 961             'info_dict': {
 962                 'id': 'Z4Vy8R84T1U',
 963                 'ext': 'mp4',
 964                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
 965                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
 966                 'duration': 433,
 967                 'upload_date': '20130923',
 968                 'uploader': 'Amelia Putri Harwita',
 969                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
 970                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
 971                 'formats': 'maxcount:10',
 972             },
 973             'params': {
 974                 'skip_download': True,
 975                 'youtube_include_dash_manifest': False,
 976             },
 977             'skip': 'not actual anymore',
 978         },
 979         {
 980             # Youtube Music Auto-generated description
 981             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
 982             'info_dict': {
 983                 'id': 'MgNrAu2pzNs',
 984                 'ext': 'mp4',
 985                 'title': 'Voyeur Girl',
 986                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
 987                 'upload_date': '20190312',
 988                 'uploader': 'Stephen - Topic',
 989                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
 990                 'artist': 'Stephen',
 991                 'track': 'Voyeur Girl',
 992                 'album': 'it\'s too much love to know my dear',
 993                 'release_date': '20190313',
 994                 'release_year': 2019,
 995             },
 996             'params': {
 997                 'skip_download': True,
 998             },
 999         },
1000         {
1001             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1002             'only_matching': True,
1003         },
1004         {
1005             # invalid -> valid video id redirection
1006             'url': 'DJztXj2GPfl',
1007             'info_dict': {
1008                 'id': 'DJztXj2GPfk',
1009                 'ext': 'mp4',
1010                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1011                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1012                 'upload_date': '20090125',
1013                 'uploader': 'Prochorowka',
1014                 'uploader_id': 'Prochorowka',
1015                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1016                 'artist': 'Panjabi MC',
1017                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1018                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1019             },
1020             'params': {
1021                 'skip_download': True,
1022             },
1023             'skip': 'Video unavailable',
1024         },
1025         {
1026             # empty description results in an empty string
1027             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1028             'info_dict': {
1029                 'id': 'x41yOUIvK2k',
1030                 'ext': 'mp4',
1031                 'title': 'IMG 3456',
1032                 'description': '',
1033                 'upload_date': '20170613',
1034                 'uploader_id': 'ElevageOrVert',
1035                 'uploader': 'ElevageOrVert',
1036             },
1037             'params': {
1038                 'skip_download': True,
1039             },
1040         },
1041         {
1042             # with '};' inside yt initial data (see [1])
1043             # see [2] for an example with '};' inside ytInitialPlayerResponse
1044             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1045             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1046             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1047             'info_dict': {
1048                 'id': 'CHqg6qOn4no',
1049                 'ext': 'mp4',
1050                 'title': 'Part 77   Sort a list of simple types in c#',
1051                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1052                 'upload_date': '20130831',
1053                 'uploader_id': 'kudvenkat',
1054                 'uploader': 'kudvenkat',
1055             },
1056             'params': {
1057                 'skip_download': True,
1058             },
1059         },
1060         {
1061             # another example of '};' in ytInitialData
1062             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1063             'only_matching': True,
1064         },
1065         {
1066             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1067             'only_matching': True,
1068         },
1069         {
1070             # https://github.com/ytdl-org/youtube-dl/pull/28094
1071             'url': 'OtqTfy26tG0',
1072             'info_dict': {
1073                 'id': 'OtqTfy26tG0',
1074                 'ext': 'mp4',
1075                 'title': 'Burn Out',
1076                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1077                 'upload_date': '20141120',
1078                 'uploader': 'The Cinematic Orchestra - Topic',
1079                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1080                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1081                 'artist': 'The Cinematic Orchestra',
1082                 'track': 'Burn Out',
1083                 'album': 'Every Day',
1084                 'release_data': None,
1085                 'release_year': None,
1086             },
1087             'params': {
1088                 'skip_download': True,
1089             },
1090         },
1091         {
1092             # controversial video, only works with bpctr when authenticated with cookies
1093             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1094             'only_matching': True,
1095         },
1096         {
1097             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
1098             'url': 'cBvYw8_A0vQ',
1099             'info_dict': {
1100                 'id': 'cBvYw8_A0vQ',
1101                 'ext': 'mp4',
1102                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
1103                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
1104                 'upload_date': '20201120',
1105                 'uploader': 'Walk around Japan',
1106                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
1107                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
1108             },
1109             'params': {
1110                 'skip_download': True,
1111             },
1112         },
1113     ]
1114     _formats = {
1115         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1116         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1117         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1118         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1119         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1120         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1121         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1122         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1123         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1124         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1125         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1126         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1127         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1128         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1129         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1130         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1131         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1132         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1133
1134
1135         # 3D videos
1136         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1137         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1138         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1139         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1140         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1141         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1142         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1143
1144         # Apple HTTP Live Streaming
1145         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1146         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1147         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1148         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1149         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1150         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1151         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1152         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1153
1154         # DASH mp4 video
1155         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1156         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1157         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1158         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1159         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1160         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1161         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1162         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1163         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1164         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1165         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1166         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1167
1168         # Dash mp4 audio
1169         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1170         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1171         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1172         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1173         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1174         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1175         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1176
1177         # Dash webm
1178         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1179         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1180         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1181         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1182         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1183         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1184         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1185         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1186         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1187         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1188         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1189         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1190         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1191         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1192         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1193         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1194         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1195         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1196         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1197         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1198         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1199         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1200
1201         # Dash webm audio
1202         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1203         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1204
1205         # Dash webm audio with opus inside
1206         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1207         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1208         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1209
1210         # RTMP (unnamed)
1211         '_rtmp': {'protocol': 'rtmp'},
1212
1213         # av01 video only formats sometimes served with "unknown" codecs
1214         '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1215         '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1216         '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1217         '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1218     }
1219
1220     @classmethod
1221     def suitable(cls, url):
1222         # Hack for lazy extractors until more generic solution is implemented
1223         # (see #28780)
1224         from .youtube import parse_qs
1225         qs = parse_qs(url)
1226         if qs.get('list', [None])[0]:
1227             return False
1228         return super(YoutubeIE, cls).suitable(url)
1229
1230     def __init__(self, *args, **kwargs):
1231         super(YoutubeIE, self).__init__(*args, **kwargs)
1232         self._code_cache = {}
1233         self._player_cache = {}
1234
1235     def _signature_cache_id(self, example_sig):
1236         """ Return a string representation of a signature """
1237         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
1238
1239     @classmethod
1240     def _extract_player_info(cls, player_url):
1241         for player_re in cls._PLAYER_INFO_RE:
1242             id_m = re.search(player_re, player_url)
1243             if id_m:
1244                 break
1245         else:
1246             raise ExtractorError('Cannot identify player %r' % player_url)
1247         return id_m.group('id')
1248
1249     def _extract_signature_function(self, video_id, player_url, example_sig):
1250         player_id = self._extract_player_info(player_url)
1251
1252         # Read from filesystem cache
1253         func_id = 'js_%s_%s' % (
1254             player_id, self._signature_cache_id(example_sig))
1255         assert os.path.basename(func_id) == func_id
1256
1257         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
1258         if cache_spec is not None:
1259             return lambda s: ''.join(s[i] for i in cache_spec)
1260
1261         if player_id not in self._code_cache:
1262             self._code_cache[player_id] = self._download_webpage(
1263                 player_url, video_id,
1264                 note='Downloading player ' + player_id,
1265                 errnote='Download of %s failed' % player_url)
1266         code = self._code_cache[player_id]
1267         res = self._parse_sig_js(code)
1268
1269         test_string = ''.join(map(compat_chr, range(len(example_sig))))
1270         cache_res = res(test_string)
1271         cache_spec = [ord(c) for c in cache_res]
1272
1273         self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
1274         return res
1275
1276     def _print_sig_code(self, func, example_sig):
1277         def gen_sig_code(idxs):
1278             def _genslice(start, end, step):
1279                 starts = '' if start == 0 else str(start)
1280                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
1281                 steps = '' if step == 1 else (':%d' % step)
1282                 return 's[%s%s%s]' % (starts, ends, steps)
1283
1284             step = None
1285             # Quelch pyflakes warnings - start will be set when step is set
1286             start = '(Never used)'
1287             for i, prev in zip(idxs[1:], idxs[:-1]):
1288                 if step is not None:
1289                     if i - prev == step:
1290                         continue
1291                     yield _genslice(start, prev, step)
1292                     step = None
1293                     continue
1294                 if i - prev in [-1, 1]:
1295                     step = i - prev
1296                     start = prev
1297                     continue
1298                 else:
1299                     yield 's[%d]' % prev
1300             if step is None:
1301                 yield 's[%d]' % i
1302             else:
1303                 yield _genslice(start, i, step)
1304
1305         test_string = ''.join(map(compat_chr, range(len(example_sig))))
1306         cache_res = func(test_string)
1307         cache_spec = [ord(c) for c in cache_res]
1308         expr_code = ' + '.join(gen_sig_code(cache_spec))
1309         signature_id_tuple = '(%s)' % (
1310             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
1311         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
1312                 '    return %s\n') % (signature_id_tuple, expr_code)
1313         self.to_screen('Extracted signature function:\n' + code)
1314
1315     def _parse_sig_js(self, jscode):
1316         funcname = self._search_regex(
1317             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1318              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1319              r'\bm=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(h\.s\)\)',
1320              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(c\)\)',
1321              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
1322              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1323              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1324              # Obsolete patterns
1325              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1326              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
1327              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1328              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1329              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1330              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1331              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1332              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
1333             jscode, 'Initial JS player signature function name', group='sig')
1334
1335         jsi = JSInterpreter(jscode)
1336         initial_function = jsi.extract_function(funcname)
1337         return lambda s: initial_function([s])
1338
1339     def _decrypt_signature(self, s, video_id, player_url):
1340         """Turn the encrypted s field into a working signature"""
1341
1342         if player_url is None:
1343             raise ExtractorError('Cannot decrypt signature without player_url')
1344
1345         if player_url.startswith('//'):
1346             player_url = 'https:' + player_url
1347         elif not re.match(r'https?://', player_url):
1348             player_url = compat_urlparse.urljoin(
1349                 'https://www.youtube.com', player_url)
1350         try:
1351             player_id = (player_url, self._signature_cache_id(s))
1352             if player_id not in self._player_cache:
1353                 func = self._extract_signature_function(
1354                     video_id, player_url, s
1355                 )
1356                 self._player_cache[player_id] = func
1357             func = self._player_cache[player_id]
1358             if self._downloader.params.get('youtube_print_sig_code'):
1359                 self._print_sig_code(func, s)
1360             return func(s)
1361         except Exception as e:
1362             tb = traceback.format_exc()
1363             raise ExtractorError(
1364                 'Signature extraction failed: ' + tb, cause=e)
1365
1366     def _mark_watched(self, video_id, player_response):
1367         playback_url = url_or_none(try_get(
1368             player_response,
1369             lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']))
1370         if not playback_url:
1371             return
1372         parsed_playback_url = compat_urlparse.urlparse(playback_url)
1373         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
1374
1375         # cpn generation algorithm is reverse engineered from base.js.
1376         # In fact it works even with dummy cpn.
1377         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
1378         cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
1379
1380         qs.update({
1381             'ver': ['2'],
1382             'cpn': [cpn],
1383         })
1384         playback_url = compat_urlparse.urlunparse(
1385             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
1386
1387         self._download_webpage(
1388             playback_url, video_id, 'Marking watched',
1389             'Unable to mark watched', fatal=False)
1390
1391     @staticmethod
1392     def _extract_urls(webpage):
1393         # Embedded YouTube player
1394         entries = [
1395             unescapeHTML(mobj.group('url'))
1396             for mobj in re.finditer(r'''(?x)
1397             (?:
1398                 <iframe[^>]+?src=|
1399                 data-video-url=|
1400                 <embed[^>]+?src=|
1401                 embedSWF\(?:\s*|
1402                 <object[^>]+data=|
1403                 new\s+SWFObject\(
1404             )
1405             (["\'])
1406                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1407                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1408             \1''', webpage)]
1409
1410         # lazyYT YouTube embed
1411         entries.extend(list(map(
1412             unescapeHTML,
1413             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
1414
1415         # Wordpress "YouTube Video Importer" plugin
1416         matches = re.findall(r'''(?x)<div[^>]+
1417             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
1418             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
1419         entries.extend(m[-1] for m in matches)
1420
1421         return entries
1422
1423     @staticmethod
1424     def _extract_url(webpage):
1425         urls = YoutubeIE._extract_urls(webpage)
1426         return urls[0] if urls else None
1427
1428     @classmethod
1429     def extract_id(cls, url):
1430         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
1431         if mobj is None:
1432             raise ExtractorError('Invalid URL: %s' % url)
1433         video_id = mobj.group(2)
1434         return video_id
1435
1436     def _extract_chapters_from_json(self, data, video_id, duration):
1437         chapters_list = try_get(
1438             data,
1439             lambda x: x['playerOverlays']
1440                        ['playerOverlayRenderer']
1441                        ['decoratedPlayerBarRenderer']
1442                        ['decoratedPlayerBarRenderer']
1443                        ['playerBar']
1444                        ['chapteredPlayerBarRenderer']
1445                        ['chapters'],
1446             list)
1447         if not chapters_list:
1448             return
1449
1450         def chapter_time(chapter):
1451             return float_or_none(
1452                 try_get(
1453                     chapter,
1454                     lambda x: x['chapterRenderer']['timeRangeStartMillis'],
1455                     int),
1456                 scale=1000)
1457         chapters = []
1458         for next_num, chapter in enumerate(chapters_list, start=1):
1459             start_time = chapter_time(chapter)
1460             if start_time is None:
1461                 continue
1462             end_time = (chapter_time(chapters_list[next_num])
1463                         if next_num < len(chapters_list) else duration)
1464             if end_time is None:
1465                 continue
1466             title = try_get(
1467                 chapter, lambda x: x['chapterRenderer']['title']['simpleText'],
1468                 compat_str)
1469             chapters.append({
1470                 'start_time': start_time,
1471                 'end_time': end_time,
1472                 'title': title,
1473             })
1474         return chapters
1475
1476     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
1477         return self._parse_json(self._search_regex(
1478             (r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE),
1479              regex), webpage, name, default='{}'), video_id, fatal=False)
1480
1481     def _real_extract(self, url):
1482         url, smuggled_data = unsmuggle_url(url, {})
1483         video_id = self._match_id(url)
1484         base_url = self.http_scheme() + '//www.youtube.com/'
1485         webpage_url = base_url + 'watch?v=' + video_id
1486         webpage = self._download_webpage(
1487             webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
1488
1489         player_response = None
1490         if webpage:
1491             player_response = self._extract_yt_initial_variable(
1492                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
1493                 video_id, 'initial player response')
1494         if not player_response:
1495             player_response = self._call_api(
1496                 'player', {'videoId': video_id}, video_id)
1497
1498         playability_status = player_response.get('playabilityStatus') or {}
1499         if playability_status.get('reason') == 'Sign in to confirm your age':
1500             pr = self._parse_json(try_get(compat_parse_qs(
1501                 self._download_webpage(
1502                     base_url + 'get_video_info', video_id,
1503                     'Refetching age-gated info webpage',
1504                     'unable to download video info webpage', query={
1505                         'video_id': video_id,
1506                         'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1507                     }, fatal=False)),
1508                 lambda x: x['player_response'][0],
1509                 compat_str) or '{}', video_id)
1510             if pr:
1511                 player_response = pr
1512
1513         trailer_video_id = try_get(
1514             playability_status,
1515             lambda x: x['errorScreen']['playerLegacyDesktopYpcTrailerRenderer']['trailerVideoId'],
1516             compat_str)
1517         if trailer_video_id:
1518             return self.url_result(
1519                 trailer_video_id, self.ie_key(), trailer_video_id)
1520
1521         def get_text(x):
1522             if not x:
1523                 return
1524             text = x.get('simpleText')
1525             if text and isinstance(text, compat_str):
1526                 return text
1527             runs = x.get('runs')
1528             if not isinstance(runs, list):
1529                 return
1530             return ''.join([r['text'] for r in runs if isinstance(r.get('text'), compat_str)])
1531
1532         search_meta = (
1533             lambda x: self._html_search_meta(x, webpage, default=None)) \
1534             if webpage else lambda x: None
1535
1536         video_details = player_response.get('videoDetails') or {}
1537         microformat = try_get(
1538             player_response,
1539             lambda x: x['microformat']['playerMicroformatRenderer'],
1540             dict) or {}
1541         video_title = video_details.get('title') \
1542             or get_text(microformat.get('title')) \
1543             or search_meta(['og:title', 'twitter:title', 'title'])
1544         video_description = video_details.get('shortDescription')
1545
1546         if not smuggled_data.get('force_singlefeed', False):
1547             if not self._downloader.params.get('noplaylist'):
1548                 multifeed_metadata_list = try_get(
1549                     player_response,
1550                     lambda x: x['multicamera']['playerLegacyMulticameraRenderer']['metadataList'],
1551                     compat_str)
1552                 if multifeed_metadata_list:
1553                     entries = []
1554                     feed_ids = []
1555                     for feed in multifeed_metadata_list.split(','):
1556                         # Unquote should take place before split on comma (,) since textual
1557                         # fields may contain comma as well (see
1558                         # https://github.com/ytdl-org/youtube-dl/issues/8536)
1559                         feed_data = compat_parse_qs(
1560                             compat_urllib_parse_unquote_plus(feed))
1561
1562                         def feed_entry(name):
1563                             return try_get(
1564                                 feed_data, lambda x: x[name][0], compat_str)
1565
1566                         feed_id = feed_entry('id')
1567                         if not feed_id:
1568                             continue
1569                         feed_title = feed_entry('title')
1570                         title = video_title
1571                         if feed_title:
1572                             title += ' (%s)' % feed_title
1573                         entries.append({
1574                             '_type': 'url_transparent',
1575                             'ie_key': 'Youtube',
1576                             'url': smuggle_url(
1577                                 base_url + 'watch?v=' + feed_data['id'][0],
1578                                 {'force_singlefeed': True}),
1579                             'title': title,
1580                         })
1581                         feed_ids.append(feed_id)
1582                     self.to_screen(
1583                         'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1584                         % (', '.join(feed_ids), video_id))
1585                     return self.playlist_result(
1586                         entries, video_id, video_title, video_description)
1587             else:
1588                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1589
1590         formats = []
1591         itags = []
1592         itag_qualities = {}
1593         player_url = None
1594         q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
1595         streaming_data = player_response.get('streamingData') or {}
1596         streaming_formats = streaming_data.get('formats') or []
1597         streaming_formats.extend(streaming_data.get('adaptiveFormats') or [])
1598         for fmt in streaming_formats:
1599             if fmt.get('targetDurationSec') or fmt.get('drmFamilies'):
1600                 continue
1601
1602             itag = str_or_none(fmt.get('itag'))
1603             quality = fmt.get('quality')
1604             if itag and quality:
1605                 itag_qualities[itag] = quality
1606             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
1607             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
1608             # number of fragment that would subsequently requested with (`&sq=N`)
1609             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
1610                 continue
1611
1612             fmt_url = fmt.get('url')
1613             if not fmt_url:
1614                 sc = compat_parse_qs(fmt.get('signatureCipher'))
1615                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
1616                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
1617                 if not (sc and fmt_url and encrypted_sig):
1618                     continue
1619                 if not player_url:
1620                     if not webpage:
1621                         continue
1622                     player_url = self._search_regex(
1623                         r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
1624                         webpage, 'player URL', fatal=False)
1625                 if not player_url:
1626                     continue
1627                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
1628                 sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
1629                 fmt_url += '&' + sp + '=' + signature
1630
1631             if itag:
1632                 itags.append(itag)
1633             tbr = float_or_none(
1634                 fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
1635             dct = {
1636                 'asr': int_or_none(fmt.get('audioSampleRate')),
1637                 'filesize': int_or_none(fmt.get('contentLength')),
1638                 'format_id': itag,
1639                 'format_note': fmt.get('qualityLabel') or quality,
1640                 'fps': int_or_none(fmt.get('fps')),
1641                 'height': int_or_none(fmt.get('height')),
1642                 'quality': q(quality),
1643                 'tbr': tbr,
1644                 'url': fmt_url,
1645                 'width': fmt.get('width'),
1646             }
1647             mimetype = fmt.get('mimeType')
1648             if mimetype:
1649                 mobj = re.match(
1650                     r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', mimetype)
1651                 if mobj:
1652                     dct['ext'] = mimetype2ext(mobj.group(1))
1653                     dct.update(parse_codecs(mobj.group(2)))
1654             no_audio = dct.get('acodec') == 'none'
1655             no_video = dct.get('vcodec') == 'none'
1656             if no_audio:
1657                 dct['vbr'] = tbr
1658             if no_video:
1659                 dct['abr'] = tbr
1660             if no_audio or no_video:
1661                 dct['downloader_options'] = {
1662                     # Youtube throttles chunks >~10M
1663                     'http_chunk_size': 10485760,
1664                 }
1665                 if dct.get('ext'):
1666                     dct['container'] = dct['ext'] + '_dash'
1667             formats.append(dct)
1668
1669         hls_manifest_url = streaming_data.get('hlsManifestUrl')
1670         if hls_manifest_url:
1671             for f in self._extract_m3u8_formats(
1672                     hls_manifest_url, video_id, 'mp4', fatal=False):
1673                 itag = self._search_regex(
1674                     r'/itag/(\d+)', f['url'], 'itag', default=None)
1675                 if itag:
1676                     f['format_id'] = itag
1677                 formats.append(f)
1678
1679         if self._downloader.params.get('youtube_include_dash_manifest', True):
1680             dash_manifest_url = streaming_data.get('dashManifestUrl')
1681             if dash_manifest_url:
1682                 for f in self._extract_mpd_formats(
1683                         dash_manifest_url, video_id, fatal=False):
1684                     itag = f['format_id']
1685                     if itag in itags:
1686                         continue
1687                     if itag in itag_qualities:
1688                         f['quality'] = q(itag_qualities[itag])
1689                     filesize = int_or_none(self._search_regex(
1690                         r'/clen/(\d+)', f.get('fragment_base_url')
1691                         or f['url'], 'file size', default=None))
1692                     if filesize:
1693                         f['filesize'] = filesize
1694                     formats.append(f)
1695
1696         if not formats:
1697             if streaming_data.get('licenseInfos'):
1698                 raise ExtractorError(
1699                     'This video is DRM protected.', expected=True)
1700             pemr = try_get(
1701                 playability_status,
1702                 lambda x: x['errorScreen']['playerErrorMessageRenderer'],
1703                 dict) or {}
1704             reason = get_text(pemr.get('reason')) or playability_status.get('reason')
1705             subreason = pemr.get('subreason')
1706             if subreason:
1707                 subreason = clean_html(get_text(subreason))
1708                 if subreason == 'The uploader has not made this video available in your country.':
1709                     countries = microformat.get('availableCountries')
1710                     if not countries:
1711                         regions_allowed = search_meta('regionsAllowed')
1712                         countries = regions_allowed.split(',') if regions_allowed else None
1713                     self.raise_geo_restricted(
1714                         subreason, countries)
1715                 reason += '\n' + subreason
1716             if reason:
1717                 raise ExtractorError(reason, expected=True)
1718
1719         self._sort_formats(formats)
1720
1721         keywords = video_details.get('keywords') or []
1722         if not keywords and webpage:
1723             keywords = [
1724                 unescapeHTML(m.group('content'))
1725                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
1726         for keyword in keywords:
1727             if keyword.startswith('yt:stretch='):
1728                 mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
1729                 if mobj:
1730                     # NB: float is intentional for forcing float division
1731                     w, h = (float(v) for v in mobj.groups())
1732                     if w > 0 and h > 0:
1733                         ratio = w / h
1734                         for f in formats:
1735                             if f.get('vcodec') != 'none':
1736                                 f['stretched_ratio'] = ratio
1737                         break
1738
1739         thumbnails = []
1740         for container in (video_details, microformat):
1741             for thumbnail in (try_get(
1742                     container,
1743                     lambda x: x['thumbnail']['thumbnails'], list) or []):
1744                 thumbnail_url = thumbnail.get('url')
1745                 if not thumbnail_url:
1746                     continue
1747                 thumbnails.append({
1748                     'height': int_or_none(thumbnail.get('height')),
1749                     'url': thumbnail_url,
1750                     'width': int_or_none(thumbnail.get('width')),
1751                 })
1752             if thumbnails:
1753                 break
1754         else:
1755             thumbnail = search_meta(['og:image', 'twitter:image'])
1756             if thumbnail:
1757                 thumbnails = [{'url': thumbnail}]
1758
1759         category = microformat.get('category') or search_meta('genre')
1760         channel_id = video_details.get('channelId') \
1761             or microformat.get('externalChannelId') \
1762             or search_meta('channelId')
1763         duration = int_or_none(
1764             video_details.get('lengthSeconds')
1765             or microformat.get('lengthSeconds')) \
1766             or parse_duration(search_meta('duration'))
1767         is_live = video_details.get('isLive')
1768         owner_profile_url = microformat.get('ownerProfileUrl')
1769
1770         info = {
1771             'id': video_id,
1772             'title': self._live_title(video_title) if is_live else video_title,
1773             'formats': formats,
1774             'thumbnails': thumbnails,
1775             'description': video_description,
1776             'upload_date': unified_strdate(
1777                 microformat.get('uploadDate')
1778                 or search_meta('uploadDate')),
1779             'uploader': video_details['author'],
1780             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
1781             'uploader_url': owner_profile_url,
1782             'channel_id': channel_id,
1783             'channel_url': 'https://www.youtube.com/channel/' + channel_id if channel_id else None,
1784             'duration': duration,
1785             'view_count': int_or_none(
1786                 video_details.get('viewCount')
1787                 or microformat.get('viewCount')
1788                 or search_meta('interactionCount')),
1789             'average_rating': float_or_none(video_details.get('averageRating')),
1790             'age_limit': 18 if (
1791                 microformat.get('isFamilySafe') is False
1792                 or search_meta('isFamilyFriendly') == 'false'
1793                 or search_meta('og:restrictions:age') == '18+') else 0,
1794             'webpage_url': webpage_url,
1795             'categories': [category] if category else None,
1796             'tags': keywords,
1797             'is_live': is_live,
1798         }
1799
1800         pctr = try_get(
1801             player_response,
1802             lambda x: x['captions']['playerCaptionsTracklistRenderer'], dict)
1803         if pctr:
1804             def process_language(container, base_url, lang_code, query):
1805                 lang_subs = []
1806                 for fmt in self._SUBTITLE_FORMATS:
1807                     query.update({
1808                         'fmt': fmt,
1809                     })
1810                     lang_subs.append({
1811                         'ext': fmt,
1812                         'url': update_url_query(base_url, query),
1813                     })
1814                 container[lang_code] = lang_subs
1815
1816             subtitles = {}
1817             for caption_track in (pctr.get('captionTracks') or []):
1818                 base_url = caption_track.get('baseUrl')
1819                 if not base_url:
1820                     continue
1821                 if caption_track.get('kind') != 'asr':
1822                     lang_code = caption_track.get('languageCode')
1823                     if not lang_code:
1824                         continue
1825                     process_language(
1826                         subtitles, base_url, lang_code, {})
1827                     continue
1828                 automatic_captions = {}
1829                 for translation_language in (pctr.get('translationLanguages') or []):
1830                     translation_language_code = translation_language.get('languageCode')
1831                     if not translation_language_code:
1832                         continue
1833                     process_language(
1834                         automatic_captions, base_url, translation_language_code,
1835                         {'tlang': translation_language_code})
1836                 info['automatic_captions'] = automatic_captions
1837             info['subtitles'] = subtitles
1838
1839         parsed_url = compat_urllib_parse_urlparse(url)
1840         for component in [parsed_url.fragment, parsed_url.query]:
1841             query = compat_parse_qs(component)
1842             for k, v in query.items():
1843                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
1844                     d_k += '_time'
1845                     if d_k not in info and k in s_ks:
1846                         info[d_k] = parse_duration(query[k][0])
1847
1848         if video_description:
1849             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
1850             if mobj:
1851                 release_year = mobj.group('release_year')
1852                 release_date = mobj.group('release_date')
1853                 if release_date:
1854                     release_date = release_date.replace('-', '')
1855                     if not release_year:
1856                         release_year = release_date[:4]
1857                 info.update({
1858                     'album': mobj.group('album'.strip()),
1859                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
1860                     'track': mobj.group('track').strip(),
1861                     'release_date': release_date,
1862                     'release_year': int_or_none(release_year),
1863                 })
1864
1865         initial_data = None
1866         if webpage:
1867             initial_data = self._extract_yt_initial_variable(
1868                 webpage, self._YT_INITIAL_DATA_RE, video_id,
1869                 'yt initial data')
1870         if not initial_data:
1871             initial_data = self._call_api(
1872                 'next', {'videoId': video_id}, video_id, fatal=False)
1873
1874         if initial_data:
1875             chapters = self._extract_chapters_from_json(
1876                 initial_data, video_id, duration)
1877             if not chapters:
1878                 for engagment_pannel in (initial_data.get('engagementPanels') or []):
1879                     contents = try_get(
1880                         engagment_pannel, lambda x: x['engagementPanelSectionListRenderer']['content']['macroMarkersListRenderer']['contents'],
1881                         list)
1882                     if not contents:
1883                         continue
1884
1885                     def chapter_time(mmlir):
1886                         return parse_duration(
1887                             get_text(mmlir.get('timeDescription')))
1888
1889                     chapters = []
1890                     for next_num, content in enumerate(contents, start=1):
1891                         mmlir = content.get('macroMarkersListItemRenderer') or {}
1892                         start_time = chapter_time(mmlir)
1893                         end_time = chapter_time(try_get(
1894                             contents, lambda x: x[next_num]['macroMarkersListItemRenderer'])) \
1895                             if next_num < len(contents) else duration
1896                         if start_time is None or end_time is None:
1897                             continue
1898                         chapters.append({
1899                             'start_time': start_time,
1900                             'end_time': end_time,
1901                             'title': get_text(mmlir.get('title')),
1902                         })
1903                     if chapters:
1904                         break
1905             if chapters:
1906                 info['chapters'] = chapters
1907
1908             contents = try_get(
1909                 initial_data,
1910                 lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'],
1911                 list) or []
1912             for content in contents:
1913                 vpir = content.get('videoPrimaryInfoRenderer')
1914                 if vpir:
1915                     stl = vpir.get('superTitleLink')
1916                     if stl:
1917                         stl = get_text(stl)
1918                         if try_get(
1919                                 vpir,
1920                                 lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
1921                             info['location'] = stl
1922                         else:
1923                             mobj = re.search(r'(.+?)\s*S(\d+)\s*•\s*E(\d+)', stl)
1924                             if mobj:
1925                                 info.update({
1926                                     'series': mobj.group(1),
1927                                     'season_number': int(mobj.group(2)),
1928                                     'episode_number': int(mobj.group(3)),
1929                                 })
1930                     for tlb in (try_get(
1931                             vpir,
1932                             lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
1933                             list) or []):
1934                         tbr = tlb.get('toggleButtonRenderer') or {}
1935                         for getter, regex in [(
1936                                 lambda x: x['defaultText']['accessibility']['accessibilityData'],
1937                                 r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
1938                                     lambda x: x['accessibility'],
1939                                     lambda x: x['accessibilityData']['accessibilityData'],
1940                                 ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
1941                             label = (try_get(tbr, getter, dict) or {}).get('label')
1942                             if label:
1943                                 mobj = re.match(regex, label)
1944                                 if mobj:
1945                                     info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
1946                                     break
1947                     sbr_tooltip = try_get(
1948                         vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
1949                     if sbr_tooltip:
1950                         like_count, dislike_count = sbr_tooltip.split(' / ')
1951                         info.update({
1952                             'like_count': str_to_int(like_count),
1953                             'dislike_count': str_to_int(dislike_count),
1954                         })
1955                 vsir = content.get('videoSecondaryInfoRenderer')
1956                 if vsir:
1957                     info['channel'] = get_text(try_get(
1958                         vsir,
1959                         lambda x: x['owner']['videoOwnerRenderer']['title'],
1960                         dict))
1961                     rows = try_get(
1962                         vsir,
1963                         lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
1964                         list) or []
1965                     multiple_songs = False
1966                     for row in rows:
1967                         if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
1968                             multiple_songs = True
1969                             break
1970                     for row in rows:
1971                         mrr = row.get('metadataRowRenderer') or {}
1972                         mrr_title = mrr.get('title')
1973                         if not mrr_title:
1974                             continue
1975                         mrr_title = get_text(mrr['title'])
1976                         mrr_contents_text = get_text(mrr['contents'][0])
1977                         if mrr_title == 'License':
1978                             info['license'] = mrr_contents_text
1979                         elif not multiple_songs:
1980                             if mrr_title == 'Album':
1981                                 info['album'] = mrr_contents_text
1982                             elif mrr_title == 'Artist':
1983                                 info['artist'] = mrr_contents_text
1984                             elif mrr_title == 'Song':
1985                                 info['track'] = mrr_contents_text
1986
1987         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
1988             v = info.get(s_k)
1989             if v:
1990                 info[d_k] = v
1991
1992         self.mark_watched(video_id, player_response)
1993
1994         return info
1995
1996
1997 class YoutubeTabIE(YoutubeBaseInfoExtractor):
1998     IE_DESC = 'YouTube.com tab'
1999     _VALID_URL = r'''(?x)
2000                     https?://
2001                         (?:\w+\.)?
2002                         (?:
2003                             youtube(?:kids)?\.com|
2004                             invidio\.us
2005                         )/
2006                         (?:
2007                             (?:channel|c|user|feed|hashtag)/|
2008                             (?:playlist|watch)\?.*?\blist=|
2009                             (?!(?:watch|embed|v|e)\b)
2010                         )
2011                         (?P<id>[^/?\#&]+)
2012                     '''
2013     IE_NAME = 'youtube:tab'
2014
2015     _TESTS = [{
2016         # playlists, multipage
2017         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
2018         'playlist_mincount': 94,
2019         'info_dict': {
2020             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
2021             'title': 'Игорь Клейнер - Playlists',
2022             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
2023         },
2024     }, {
2025         # playlists, multipage, different order
2026         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
2027         'playlist_mincount': 94,
2028         'info_dict': {
2029             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
2030             'title': 'Игорь Клейнер - Playlists',
2031             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
2032         },
2033     }, {
2034         # playlists, series
2035         'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
2036         'playlist_mincount': 5,
2037         'info_dict': {
2038             'id': 'UCYO_jab_esuFRV4b17AJtAw',
2039             'title': '3Blue1Brown - Playlists',
2040             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
2041         },
2042     }, {
2043         # playlists, singlepage
2044         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
2045         'playlist_mincount': 4,
2046         'info_dict': {
2047             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
2048             'title': 'ThirstForScience - Playlists',
2049             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
2050         }
2051     }, {
2052         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
2053         'only_matching': True,
2054     }, {
2055         # basic, single video playlist
2056         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2057         'info_dict': {
2058             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
2059             'uploader': 'Sergey M.',
2060             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2061             'title': 'youtube-dl public playlist',
2062         },
2063         'playlist_count': 1,
2064     }, {
2065         # empty playlist
2066         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
2067         'info_dict': {
2068             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
2069             'uploader': 'Sergey M.',
2070             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
2071             'title': 'youtube-dl empty playlist',
2072         },
2073         'playlist_count': 0,
2074     }, {
2075         # Home tab
2076         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
2077         'info_dict': {
2078             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2079             'title': 'lex will - Home',
2080             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2081         },
2082         'playlist_mincount': 2,
2083     }, {
2084         # Videos tab
2085         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
2086         'info_dict': {
2087             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2088             'title': 'lex will - Videos',
2089             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2090         },
2091         'playlist_mincount': 975,
2092     }, {
2093         # Videos tab, sorted by popular
2094         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
2095         'info_dict': {
2096             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2097             'title': 'lex will - Videos',
2098             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2099         },
2100         'playlist_mincount': 199,
2101     }, {
2102         # Playlists tab
2103         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
2104         'info_dict': {
2105             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2106             'title': 'lex will - Playlists',
2107             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2108         },
2109         'playlist_mincount': 17,
2110     }, {
2111         # Community tab
2112         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
2113         'info_dict': {
2114             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2115             'title': 'lex will - Community',
2116             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2117         },
2118         'playlist_mincount': 18,
2119     }, {
2120         # Channels tab
2121         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
2122         'info_dict': {
2123             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2124             'title': 'lex will - Channels',
2125             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2126         },
2127         'playlist_mincount': 138,
2128     }, {
2129         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
2130         'only_matching': True,
2131     }, {
2132         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
2133         'only_matching': True,
2134     }, {
2135         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
2136         'only_matching': True,
2137     }, {
2138         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
2139         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2140         'info_dict': {
2141             'title': '29C3: Not my department',
2142             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2143             'uploader': 'Christiaan008',
2144             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
2145         },
2146         'playlist_count': 96,
2147     }, {
2148         'note': 'Large playlist',
2149         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
2150         'info_dict': {
2151             'title': 'Uploads from Cauchemar',
2152             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
2153             'uploader': 'Cauchemar',
2154             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
2155         },
2156         'playlist_mincount': 1123,
2157     }, {
2158         # even larger playlist, 8832 videos
2159         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
2160         'only_matching': True,
2161     }, {
2162         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
2163         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
2164         'info_dict': {
2165             'title': 'Uploads from Interstellar Movie',
2166             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
2167             'uploader': 'Interstellar Movie',
2168             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
2169         },
2170         'playlist_mincount': 21,
2171     }, {
2172         # https://github.com/ytdl-org/youtube-dl/issues/21844
2173         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2174         'info_dict': {
2175             'title': 'Data Analysis with Dr Mike Pound',
2176             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2177             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
2178             'uploader': 'Computerphile',
2179         },
2180         'playlist_mincount': 11,
2181     }, {
2182         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2183         'only_matching': True,
2184     }, {
2185         # Playlist URL that does not actually serve a playlist
2186         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
2187         'info_dict': {
2188             'id': 'FqZTN594JQw',
2189             'ext': 'webm',
2190             'title': "Smiley's People 01 detective, Adventure Series, Action",
2191             'uploader': 'STREEM',
2192             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
2193             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
2194             'upload_date': '20150526',
2195             'license': 'Standard YouTube License',
2196             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
2197             'categories': ['People & Blogs'],
2198             'tags': list,
2199             'view_count': int,
2200             'like_count': int,
2201             'dislike_count': int,
2202         },
2203         'params': {
2204             'skip_download': True,
2205         },
2206         'skip': 'This video is not available.',
2207         'add_ie': [YoutubeIE.ie_key()],
2208     }, {
2209         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
2210         'only_matching': True,
2211     }, {
2212         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
2213         'only_matching': True,
2214     }, {
2215         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
2216         'info_dict': {
2217             'id': '9Auq9mYxFEE',
2218             'ext': 'mp4',
2219             'title': 'Watch Sky News live',
2220             'uploader': 'Sky News',
2221             'uploader_id': 'skynews',
2222             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
2223             'upload_date': '20191102',
2224             'description': 'md5:78de4e1c2359d0ea3ed829678e38b662',
2225             'categories': ['News & Politics'],
2226             'tags': list,
2227             'like_count': int,
2228             'dislike_count': int,
2229         },
2230         'params': {
2231             'skip_download': True,
2232         },
2233     }, {
2234         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
2235         'info_dict': {
2236             'id': 'a48o2S1cPoo',
2237             'ext': 'mp4',
2238             'title': 'The Young Turks - Live Main Show',
2239             'uploader': 'The Young Turks',
2240             'uploader_id': 'TheYoungTurks',
2241             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
2242             'upload_date': '20150715',
2243             'license': 'Standard YouTube License',
2244             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
2245             'categories': ['News & Politics'],
2246             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
2247             'like_count': int,
2248             'dislike_count': int,
2249         },
2250         'params': {
2251             'skip_download': True,
2252         },
2253         'only_matching': True,
2254     }, {
2255         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
2256         'only_matching': True,
2257     }, {
2258         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
2259         'only_matching': True,
2260     }, {
2261         'url': 'https://www.youtube.com/feed/trending',
2262         'only_matching': True,
2263     }, {
2264         # needs auth
2265         'url': 'https://www.youtube.com/feed/library',
2266         'only_matching': True,
2267     }, {
2268         # needs auth
2269         'url': 'https://www.youtube.com/feed/history',
2270         'only_matching': True,
2271     }, {
2272         # needs auth
2273         'url': 'https://www.youtube.com/feed/subscriptions',
2274         'only_matching': True,
2275     }, {
2276         # needs auth
2277         'url': 'https://www.youtube.com/feed/watch_later',
2278         'only_matching': True,
2279     }, {
2280         # no longer available?
2281         'url': 'https://www.youtube.com/feed/recommended',
2282         'only_matching': True,
2283     }, {
2284         # inline playlist with not always working continuations
2285         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
2286         'only_matching': True,
2287     }, {
2288         'url': 'https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8',
2289         'only_matching': True,
2290     }, {
2291         'url': 'https://www.youtube.com/course',
2292         'only_matching': True,
2293     }, {
2294         'url': 'https://www.youtube.com/zsecurity',
2295         'only_matching': True,
2296     }, {
2297         'url': 'http://www.youtube.com/NASAgovVideo/videos',
2298         'only_matching': True,
2299     }, {
2300         'url': 'https://www.youtube.com/TheYoungTurks/live',
2301         'only_matching': True,
2302     }, {
2303         'url': 'https://www.youtube.com/hashtag/cctv9',
2304         'info_dict': {
2305             'id': 'cctv9',
2306             'title': '#cctv9',
2307         },
2308         'playlist_mincount': 350,
2309     }, {
2310         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
2311         'only_matching': True,
2312     }]
2313
2314     @classmethod
2315     def suitable(cls, url):
2316         return False if YoutubeIE.suitable(url) else super(
2317             YoutubeTabIE, cls).suitable(url)
2318
2319     def _extract_channel_id(self, webpage):
2320         channel_id = self._html_search_meta(
2321             'channelId', webpage, 'channel id', default=None)
2322         if channel_id:
2323             return channel_id
2324         channel_url = self._html_search_meta(
2325             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
2326              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
2327              'twitter:app:url:googleplay'), webpage, 'channel url')
2328         return self._search_regex(
2329             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
2330             channel_url, 'channel id')
2331
2332     @staticmethod
2333     def _extract_grid_item_renderer(item):
2334         assert isinstance(item, dict)
2335         for key, renderer in item.items():
2336             if not key.startswith('grid') or not key.endswith('Renderer'):
2337                 continue
2338             if not isinstance(renderer, dict):
2339                 continue
2340             return renderer
2341
2342     def _grid_entries(self, grid_renderer):
2343         for item in grid_renderer['items']:
2344             if not isinstance(item, dict):
2345                 continue
2346             renderer = self._extract_grid_item_renderer(item)
2347             if not isinstance(renderer, dict):
2348                 continue
2349             title = try_get(
2350                 renderer, (lambda x: x['title']['runs'][0]['text'],
2351                            lambda x: x['title']['simpleText']), compat_str)
2352             # playlist
2353             playlist_id = renderer.get('playlistId')
2354             if playlist_id:
2355                 yield self.url_result(
2356                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
2357                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
2358                     video_title=title)
2359                 continue
2360             # video
2361             video_id = renderer.get('videoId')
2362             if video_id:
2363                 yield self._extract_video(renderer)
2364                 continue
2365             # channel
2366             channel_id = renderer.get('channelId')
2367             if channel_id:
2368                 title = try_get(
2369                     renderer, lambda x: x['title']['simpleText'], compat_str)
2370                 yield self.url_result(
2371                     'https://www.youtube.com/channel/%s' % channel_id,
2372                     ie=YoutubeTabIE.ie_key(), video_title=title)
2373                 continue
2374             # generic endpoint URL support
2375             ep_url = urljoin('https://www.youtube.com/', try_get(
2376                 renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
2377                 compat_str))
2378             if ep_url:
2379                 for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
2380                     if ie.suitable(ep_url):
2381                         yield self.url_result(
2382                             ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
2383                         break
2384
2385     def _shelf_entries_from_content(self, shelf_renderer):
2386         content = shelf_renderer.get('content')
2387         if not isinstance(content, dict):
2388             return
2389         renderer = content.get('gridRenderer')
2390         if renderer:
2391             # TODO: add support for nested playlists so each shelf is processed
2392             # as separate playlist
2393             # TODO: this includes only first N items
2394             for entry in self._grid_entries(renderer):
2395                 yield entry
2396         renderer = content.get('horizontalListRenderer')
2397         if renderer:
2398             # TODO
2399             pass
2400
2401     def _shelf_entries(self, shelf_renderer, skip_channels=False):
2402         ep = try_get(
2403             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
2404             compat_str)
2405         shelf_url = urljoin('https://www.youtube.com', ep)
2406         if shelf_url:
2407             # Skipping links to another channels, note that checking for
2408             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
2409             # will not work
2410             if skip_channels and '/channels?' in shelf_url:
2411                 return
2412             title = try_get(
2413                 shelf_renderer, lambda x: x['title']['runs'][0]['text'], compat_str)
2414             yield self.url_result(shelf_url, video_title=title)
2415         # Shelf may not contain shelf URL, fallback to extraction from content
2416         for entry in self._shelf_entries_from_content(shelf_renderer):
2417             yield entry
2418
2419     def _playlist_entries(self, video_list_renderer):
2420         for content in video_list_renderer['contents']:
2421             if not isinstance(content, dict):
2422                 continue
2423             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
2424             if not isinstance(renderer, dict):
2425                 continue
2426             video_id = renderer.get('videoId')
2427             if not video_id:
2428                 continue
2429             yield self._extract_video(renderer)
2430
2431     def _video_entry(self, video_renderer):
2432         video_id = video_renderer.get('videoId')
2433         if video_id:
2434             return self._extract_video(video_renderer)
2435
2436     def _post_thread_entries(self, post_thread_renderer):
2437         post_renderer = try_get(
2438             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
2439         if not post_renderer:
2440             return
2441         # video attachment
2442         video_renderer = try_get(
2443             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict)
2444         video_id = None
2445         if video_renderer:
2446             entry = self._video_entry(video_renderer)
2447             if entry:
2448                 yield entry
2449         # inline video links
2450         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
2451         for run in runs:
2452             if not isinstance(run, dict):
2453                 continue
2454             ep_url = try_get(
2455                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
2456             if not ep_url:
2457                 continue
2458             if not YoutubeIE.suitable(ep_url):
2459                 continue
2460             ep_video_id = YoutubeIE._match_id(ep_url)
2461             if video_id == ep_video_id:
2462                 continue
2463             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=video_id)
2464
2465     def _post_thread_continuation_entries(self, post_thread_continuation):
2466         contents = post_thread_continuation.get('contents')
2467         if not isinstance(contents, list):
2468             return
2469         for content in contents:
2470             renderer = content.get('backstagePostThreadRenderer')
2471             if not isinstance(renderer, dict):
2472                 continue
2473             for entry in self._post_thread_entries(renderer):
2474                 yield entry
2475
2476     def _rich_grid_entries(self, contents):
2477         for content in contents:
2478             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
2479             if video_renderer:
2480                 entry = self._video_entry(video_renderer)
2481                 if entry:
2482                     yield entry
2483
2484     @staticmethod
2485     def _build_continuation_query(continuation, ctp=None):
2486         query = {
2487             'ctoken': continuation,
2488             'continuation': continuation,
2489         }
2490         if ctp:
2491             query['itct'] = ctp
2492         return query
2493
2494     @staticmethod
2495     def _extract_next_continuation_data(renderer):
2496         next_continuation = try_get(
2497             renderer, lambda x: x['continuations'][0]['nextContinuationData'], dict)
2498         if not next_continuation:
2499             return
2500         continuation = next_continuation.get('continuation')
2501         if not continuation:
2502             return
2503         ctp = next_continuation.get('clickTrackingParams')
2504         return YoutubeTabIE._build_continuation_query(continuation, ctp)
2505
2506     @classmethod
2507     def _extract_continuation(cls, renderer):
2508         next_continuation = cls._extract_next_continuation_data(renderer)
2509         if next_continuation:
2510             return next_continuation
2511         contents = []
2512         for key in ('contents', 'items'):
2513             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
2514         for content in contents:
2515             if not isinstance(content, dict):
2516                 continue
2517             continuation_ep = try_get(
2518                 content, lambda x: x['continuationItemRenderer']['continuationEndpoint'],
2519                 dict)
2520             if not continuation_ep:
2521                 continue
2522             continuation = try_get(
2523                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
2524             if not continuation:
2525                 continue
2526             ctp = continuation_ep.get('clickTrackingParams')
2527             return YoutubeTabIE._build_continuation_query(continuation, ctp)
2528
2529     def _entries(self, tab, item_id, webpage):
2530         tab_content = try_get(tab, lambda x: x['content'], dict)
2531         if not tab_content:
2532             return
2533         slr_renderer = try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
2534         if slr_renderer:
2535             is_channels_tab = tab.get('title') == 'Channels'
2536             continuation = None
2537             slr_contents = try_get(slr_renderer, lambda x: x['contents'], list) or []
2538             for slr_content in slr_contents:
2539                 if not isinstance(slr_content, dict):
2540                     continue
2541                 is_renderer = try_get(slr_content, lambda x: x['itemSectionRenderer'], dict)
2542                 if not is_renderer:
2543                     continue
2544                 isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
2545                 for isr_content in isr_contents:
2546                     if not isinstance(isr_content, dict):
2547                         continue
2548                     renderer = isr_content.get('playlistVideoListRenderer')
2549                     if renderer:
2550                         for entry in self._playlist_entries(renderer):
2551                             yield entry
2552                         continuation = self._extract_continuation(renderer)
2553                         continue
2554                     renderer = isr_content.get('gridRenderer')
2555                     if renderer:
2556                         for entry in self._grid_entries(renderer):
2557                             yield entry
2558                         continuation = self._extract_continuation(renderer)
2559                         continue
2560                     renderer = isr_content.get('shelfRenderer')
2561                     if renderer:
2562                         for entry in self._shelf_entries(renderer, not is_channels_tab):
2563                             yield entry
2564                         continue
2565                     renderer = isr_content.get('backstagePostThreadRenderer')
2566                     if renderer:
2567                         for entry in self._post_thread_entries(renderer):
2568                             yield entry
2569                         continuation = self._extract_continuation(renderer)
2570                         continue
2571                     renderer = isr_content.get('videoRenderer')
2572                     if renderer:
2573                         entry = self._video_entry(renderer)
2574                         if entry:
2575                             yield entry
2576
2577                 if not continuation:
2578                     continuation = self._extract_continuation(is_renderer)
2579             if not continuation:
2580                 continuation = self._extract_continuation(slr_renderer)
2581         else:
2582             rich_grid_renderer = tab_content.get('richGridRenderer')
2583             if not rich_grid_renderer:
2584                 return
2585             for entry in self._rich_grid_entries(rich_grid_renderer.get('contents') or []):
2586                 yield entry
2587             continuation = self._extract_continuation(rich_grid_renderer)
2588
2589         ytcfg = self._extract_ytcfg(item_id, webpage)
2590         client_version = try_get(
2591             ytcfg, lambda x: x['INNERTUBE_CLIENT_VERSION'], compat_str) or '2.20210407.08.00'
2592
2593         headers = {
2594             'x-youtube-client-name': '1',
2595             'x-youtube-client-version': client_version,
2596             'content-type': 'application/json',
2597         }
2598
2599         context = try_get(ytcfg, lambda x: x['INNERTUBE_CONTEXT'], dict) or {
2600             'client': {
2601                 'clientName': 'WEB',
2602                 'clientVersion': client_version,
2603             }
2604         }
2605         visitor_data = try_get(context, lambda x: x['client']['visitorData'], compat_str)
2606
2607         identity_token = self._extract_identity_token(ytcfg, webpage)
2608         if identity_token:
2609             headers['x-youtube-identity-token'] = identity_token
2610
2611         data = {
2612             'context': context,
2613         }
2614
2615         for page_num in itertools.count(1):
2616             if not continuation:
2617                 break
2618             if visitor_data:
2619                 headers['x-goog-visitor-id'] = visitor_data
2620             data['continuation'] = continuation['continuation']
2621             data['clickTracking'] = {
2622                 'clickTrackingParams': continuation['itct']
2623             }
2624             count = 0
2625             retries = 3
2626             while count <= retries:
2627                 try:
2628                     # Downloading page may result in intermittent 5xx HTTP error
2629                     # that is usually worked around with a retry
2630                     response = self._download_json(
2631                         'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
2632                         None, 'Downloading page %d%s' % (page_num, ' (retry #%d)' % count if count else ''),
2633                         headers=headers, data=json.dumps(data).encode('utf8'))
2634                     break
2635                 except ExtractorError as e:
2636                     if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503):
2637                         count += 1
2638                         if count <= retries:
2639                             continue
2640                     raise
2641             if not response:
2642                 break
2643
2644             visitor_data = try_get(
2645                 response, lambda x: x['responseContext']['visitorData'], compat_str) or visitor_data
2646
2647             continuation_contents = try_get(
2648                 response, lambda x: x['continuationContents'], dict)
2649             if continuation_contents:
2650                 continuation_renderer = continuation_contents.get('playlistVideoListContinuation')
2651                 if continuation_renderer:
2652                     for entry in self._playlist_entries(continuation_renderer):
2653                         yield entry
2654                     continuation = self._extract_continuation(continuation_renderer)
2655                     continue
2656                 continuation_renderer = continuation_contents.get('gridContinuation')
2657                 if continuation_renderer:
2658                     for entry in self._grid_entries(continuation_renderer):
2659                         yield entry
2660                     continuation = self._extract_continuation(continuation_renderer)
2661                     continue
2662                 continuation_renderer = continuation_contents.get('itemSectionContinuation')
2663                 if continuation_renderer:
2664                     for entry in self._post_thread_continuation_entries(continuation_renderer):
2665                         yield entry
2666                     continuation = self._extract_continuation(continuation_renderer)
2667                     continue
2668
2669             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
2670             continuation_items = try_get(
2671                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
2672             if continuation_items:
2673                 continuation_item = continuation_items[0]
2674                 if not isinstance(continuation_item, dict):
2675                     continue
2676                 renderer = self._extract_grid_item_renderer(continuation_item)
2677                 if renderer:
2678                     grid_renderer = {'items': continuation_items}
2679                     for entry in self._grid_entries(grid_renderer):
2680                         yield entry
2681                     continuation = self._extract_continuation(grid_renderer)
2682                     continue
2683                 renderer = continuation_item.get('playlistVideoRenderer') or continuation_item.get('itemSectionRenderer')
2684                 if renderer:
2685                     video_list_renderer = {'contents': continuation_items}
2686                     for entry in self._playlist_entries(video_list_renderer):
2687                         yield entry
2688                     continuation = self._extract_continuation(video_list_renderer)
2689                     continue
2690                 renderer = continuation_item.get('backstagePostThreadRenderer')
2691                 if renderer:
2692                     continuation_renderer = {'contents': continuation_items}
2693                     for entry in self._post_thread_continuation_entries(continuation_renderer):
2694                         yield entry
2695                     continuation = self._extract_continuation(continuation_renderer)
2696                     continue
2697                 renderer = continuation_item.get('richItemRenderer')
2698                 if renderer:
2699                     for entry in self._rich_grid_entries(continuation_items):
2700                         yield entry
2701                     continuation = self._extract_continuation({'contents': continuation_items})
2702                     continue
2703
2704             break
2705
2706     @staticmethod
2707     def _extract_selected_tab(tabs):
2708         for tab in tabs:
2709             if try_get(tab, lambda x: x['tabRenderer']['selected'], bool):
2710                 return tab['tabRenderer']
2711         else:
2712             raise ExtractorError('Unable to find selected tab')
2713
2714     @staticmethod
2715     def _extract_uploader(data):
2716         uploader = {}
2717         sidebar_renderer = try_get(
2718             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list)
2719         if sidebar_renderer:
2720             for item in sidebar_renderer:
2721                 if not isinstance(item, dict):
2722                     continue
2723                 renderer = item.get('playlistSidebarSecondaryInfoRenderer')
2724                 if not isinstance(renderer, dict):
2725                     continue
2726                 owner = try_get(
2727                     renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
2728                 if owner:
2729                     uploader['uploader'] = owner.get('text')
2730                     uploader['uploader_id'] = try_get(
2731                         owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
2732                     uploader['uploader_url'] = urljoin(
2733                         'https://www.youtube.com/',
2734                         try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
2735         return uploader
2736
2737     @staticmethod
2738     def _extract_alert(data):
2739         alerts = []
2740         for alert in try_get(data, lambda x: x['alerts'], list) or []:
2741             if not isinstance(alert, dict):
2742                 continue
2743             alert_text = try_get(
2744                 alert, lambda x: x['alertRenderer']['text'], dict)
2745             if not alert_text:
2746                 continue
2747             text = try_get(
2748                 alert_text,
2749                 (lambda x: x['simpleText'], lambda x: x['runs'][0]['text']),
2750                 compat_str)
2751             if text:
2752                 alerts.append(text)
2753         return '\n'.join(alerts)
2754
2755     def _extract_from_tabs(self, item_id, webpage, data, tabs):
2756         selected_tab = self._extract_selected_tab(tabs)
2757         renderer = try_get(
2758             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
2759         playlist_id = item_id
2760         title = description = None
2761         if renderer:
2762             channel_title = renderer.get('title') or item_id
2763             tab_title = selected_tab.get('title')
2764             title = channel_title or item_id
2765             if tab_title:
2766                 title += ' - %s' % tab_title
2767             description = renderer.get('description')
2768             playlist_id = renderer.get('externalId')
2769         else:
2770             renderer = try_get(
2771                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
2772             if renderer:
2773                 title = renderer.get('title')
2774             else:
2775                 renderer = try_get(
2776                     data, lambda x: x['header']['hashtagHeaderRenderer'], dict)
2777                 if renderer:
2778                     title = try_get(renderer, lambda x: x['hashtag']['simpleText'])
2779         playlist = self.playlist_result(
2780             self._entries(selected_tab, item_id, webpage),
2781             playlist_id=playlist_id, playlist_title=title,
2782             playlist_description=description)
2783         playlist.update(self._extract_uploader(data))
2784         return playlist
2785
2786     def _extract_from_playlist(self, item_id, url, data, playlist):
2787         title = playlist.get('title') or try_get(
2788             data, lambda x: x['titleText']['simpleText'], compat_str)
2789         playlist_id = playlist.get('playlistId') or item_id
2790         # Inline playlist rendition continuation does not always work
2791         # at Youtube side, so delegating regular tab-based playlist URL
2792         # processing whenever possible.
2793         playlist_url = urljoin(url, try_get(
2794             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
2795             compat_str))
2796         if playlist_url and playlist_url != url:
2797             return self.url_result(
2798                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
2799                 video_title=title)
2800         return self.playlist_result(
2801             self._playlist_entries(playlist), playlist_id=playlist_id,
2802             playlist_title=title)
2803
2804     def _extract_identity_token(self, ytcfg, webpage):
2805         if ytcfg:
2806             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
2807             if token:
2808                 return token
2809         return self._search_regex(
2810             r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
2811             'identity token', default=None)
2812
2813     def _real_extract(self, url):
2814         item_id = self._match_id(url)
2815         url = compat_urlparse.urlunparse(
2816             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
2817         # Handle both video/playlist URLs
2818         qs = parse_qs(url)
2819         video_id = qs.get('v', [None])[0]
2820         playlist_id = qs.get('list', [None])[0]
2821         if video_id and playlist_id:
2822             if self._downloader.params.get('noplaylist'):
2823                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
2824                 return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
2825             self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
2826         webpage = self._download_webpage(url, item_id)
2827         data = self._extract_yt_initial_data(item_id, webpage)
2828         tabs = try_get(
2829             data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
2830         if tabs:
2831             return self._extract_from_tabs(item_id, webpage, data, tabs)
2832         playlist = try_get(
2833             data, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
2834         if playlist:
2835             return self._extract_from_playlist(item_id, url, data, playlist)
2836         # Fallback to video extraction if no playlist alike page is recognized.
2837         # First check for the current video then try the v attribute of URL query.
2838         video_id = try_get(
2839             data, lambda x: x['currentVideoEndpoint']['watchEndpoint']['videoId'],
2840             compat_str) or video_id
2841         if video_id:
2842             return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
2843         # Capture and output alerts
2844         alert = self._extract_alert(data)
2845         if alert:
2846             raise ExtractorError(alert, expected=True)
2847         # Failed to recognize
2848         raise ExtractorError('Unable to recognize tab page')
2849
2850
2851 class YoutubePlaylistIE(InfoExtractor):
2852     IE_DESC = 'YouTube.com playlists'
2853     _VALID_URL = r'''(?x)(?:
2854                         (?:https?://)?
2855                         (?:\w+\.)?
2856                         (?:
2857                             (?:
2858                                 youtube(?:kids)?\.com|
2859                                 invidio\.us
2860                             )
2861                             /.*?\?.*?\blist=
2862                         )?
2863                         (?P<id>%(playlist_id)s)
2864                      )''' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
2865     IE_NAME = 'youtube:playlist'
2866     _TESTS = [{
2867         'note': 'issue #673',
2868         'url': 'PLBB231211A4F62143',
2869         'info_dict': {
2870             'title': '[OLD]Team Fortress 2 (Class-based LP)',
2871             'id': 'PLBB231211A4F62143',
2872             'uploader': 'Wickydoo',
2873             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
2874         },
2875         'playlist_mincount': 29,
2876     }, {
2877         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2878         'info_dict': {
2879             'title': 'YDL_safe_search',
2880             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2881         },
2882         'playlist_count': 2,
2883         'skip': 'This playlist is private',
2884     }, {
2885         'note': 'embedded',
2886         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2887         'playlist_count': 4,
2888         'info_dict': {
2889             'title': 'JODA15',
2890             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2891             'uploader': 'milan',
2892             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
2893         }
2894     }, {
2895         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2896         'playlist_mincount': 982,
2897         'info_dict': {
2898             'title': '2018 Chinese New Singles (11/6 updated)',
2899             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2900             'uploader': 'LBK',
2901             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
2902         }
2903     }, {
2904         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
2905         'only_matching': True,
2906     }, {
2907         # music album playlist
2908         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
2909         'only_matching': True,
2910     }]
2911
2912     @classmethod
2913     def suitable(cls, url):
2914         if YoutubeTabIE.suitable(url):
2915             return False
2916         # Hack for lazy extractors until more generic solution is implemented
2917         # (see #28780)
2918         from .youtube import parse_qs
2919         qs = parse_qs(url)
2920         if qs.get('v', [None])[0]:
2921             return False
2922         return super(YoutubePlaylistIE, cls).suitable(url)
2923
2924     def _real_extract(self, url):
2925         playlist_id = self._match_id(url)
2926         qs = parse_qs(url)
2927         if not qs:
2928             qs = {'list': playlist_id}
2929         return self.url_result(
2930             update_url_query('https://www.youtube.com/playlist', qs),
2931             ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
2932
2933
2934 class YoutubeYtBeIE(InfoExtractor):
2935     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
2936     _TESTS = [{
2937         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
2938         'info_dict': {
2939             'id': 'yeWKywCrFtk',
2940             'ext': 'mp4',
2941             'title': 'Small Scale Baler and Braiding Rugs',
2942             'uploader': 'Backus-Page House Museum',
2943             'uploader_id': 'backuspagemuseum',
2944             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
2945             'upload_date': '20161008',
2946             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
2947             'categories': ['Nonprofits & Activism'],
2948             'tags': list,
2949             'like_count': int,
2950             'dislike_count': int,
2951         },
2952         'params': {
2953             'noplaylist': True,
2954             'skip_download': True,
2955         },
2956     }, {
2957         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
2958         'only_matching': True,
2959     }]
2960
2961     def _real_extract(self, url):
2962         mobj = re.match(self._VALID_URL, url)
2963         video_id = mobj.group('id')
2964         playlist_id = mobj.group('playlist_id')
2965         return self.url_result(
2966             update_url_query('https://www.youtube.com/watch', {
2967                 'v': video_id,
2968                 'list': playlist_id,
2969                 'feature': 'youtu.be',
2970             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
2971
2972
2973 class YoutubeYtUserIE(InfoExtractor):
2974     _VALID_URL = r'ytuser:(?P<id>.+)'
2975     _TESTS = [{
2976         'url': 'ytuser:phihag',
2977         'only_matching': True,
2978     }]
2979
2980     def _real_extract(self, url):
2981         user_id = self._match_id(url)
2982         return self.url_result(
2983             'https://www.youtube.com/user/%s' % user_id,
2984             ie=YoutubeTabIE.ie_key(), video_id=user_id)
2985
2986
2987 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
2988     IE_NAME = 'youtube:favorites'
2989     IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
2990     _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
2991     _LOGIN_REQUIRED = True
2992     _TESTS = [{
2993         'url': ':ytfav',
2994         'only_matching': True,
2995     }, {
2996         'url': ':ytfavorites',
2997         'only_matching': True,
2998     }]
2999
3000     def _real_extract(self, url):
3001         return self.url_result(
3002             'https://www.youtube.com/playlist?list=LL',
3003             ie=YoutubeTabIE.ie_key())
3004
3005
3006 class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
3007     IE_DESC = 'YouTube.com searches'
3008     # there doesn't appear to be a real limit, for example if you search for
3009     # 'python' you get more than 8.000.000 results
3010     _MAX_RESULTS = float('inf')
3011     IE_NAME = 'youtube:search'
3012     _SEARCH_KEY = 'ytsearch'
3013     _SEARCH_PARAMS = None
3014     _TESTS = []
3015
3016     def _entries(self, query, n):
3017         data = {
3018             'context': {
3019                 'client': {
3020                     'clientName': 'WEB',
3021                     'clientVersion': '2.20201021.03.00',
3022                 }
3023             },
3024             'query': query,
3025         }
3026         if self._SEARCH_PARAMS:
3027             data['params'] = self._SEARCH_PARAMS
3028         total = 0
3029         for page_num in itertools.count(1):
3030             search = self._download_json(
3031                 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
3032                 video_id='query "%s"' % query,
3033                 note='Downloading page %s' % page_num,
3034                 errnote='Unable to download API page', fatal=False,
3035                 data=json.dumps(data).encode('utf8'),
3036                 headers={'content-type': 'application/json'})
3037             if not search:
3038                 break
3039             slr_contents = try_get(
3040                 search,
3041                 (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
3042                  lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
3043                 list)
3044             if not slr_contents:
3045                 break
3046             for slr_content in slr_contents:
3047                 isr_contents = try_get(
3048                     slr_content,
3049                     lambda x: x['itemSectionRenderer']['contents'],
3050                     list)
3051                 if not isr_contents:
3052                     continue
3053                 for content in isr_contents:
3054                     if not isinstance(content, dict):
3055                         continue
3056                     video = content.get('videoRenderer')
3057                     if not isinstance(video, dict):
3058                         continue
3059                     video_id = video.get('videoId')
3060                     if not video_id:
3061                         continue
3062                     yield self._extract_video(video)
3063                     total += 1
3064                     if total == n:
3065                         return
3066             token = try_get(
3067                 slr_contents,
3068                 lambda x: x[-1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'],
3069                 compat_str)
3070             if not token:
3071                 break
3072             data['continuation'] = token
3073
3074     def _get_n_results(self, query, n):
3075         """Get a specified number of results for a query"""
3076         return self.playlist_result(self._entries(query, n), query)
3077
3078
3079 class YoutubeSearchDateIE(YoutubeSearchIE):
3080     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
3081     _SEARCH_KEY = 'ytsearchdate'
3082     IE_DESC = 'YouTube.com searches, newest videos first'
3083     _SEARCH_PARAMS = 'CAI%3D'
3084
3085
3086 r"""
3087 class YoutubeSearchURLIE(YoutubeSearchIE):
3088     IE_DESC = 'YouTube.com search URLs'
3089     IE_NAME = 'youtube:search_url'
3090     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
3091     _TESTS = [{
3092         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
3093         'playlist_mincount': 5,
3094         'info_dict': {
3095             'title': 'youtube-dl test video',
3096         }
3097     }, {
3098         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
3099         'only_matching': True,
3100     }]
3101
3102     def _real_extract(self, url):
3103         mobj = re.match(self._VALID_URL, url)
3104         query = compat_urllib_parse_unquote_plus(mobj.group('query'))
3105         webpage = self._download_webpage(url, query)
3106         return self.playlist_result(self._process_page(webpage), playlist_title=query)
3107 """
3108
3109
3110 class YoutubeFeedsInfoExtractor(YoutubeTabIE):
3111     """
3112     Base class for feed extractors
3113     Subclasses must define the _FEED_NAME property.
3114     """
3115     _LOGIN_REQUIRED = True
3116
3117     @property
3118     def IE_NAME(self):
3119         return 'youtube:%s' % self._FEED_NAME
3120
3121     def _real_initialize(self):
3122         self._login()
3123
3124     def _real_extract(self, url):
3125         return self.url_result(
3126             'https://www.youtube.com/feed/%s' % self._FEED_NAME,
3127             ie=YoutubeTabIE.ie_key())
3128
3129
3130 class YoutubeWatchLaterIE(InfoExtractor):
3131     IE_NAME = 'youtube:watchlater'
3132     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
3133     _VALID_URL = r':ytwatchlater'
3134     _TESTS = [{
3135         'url': ':ytwatchlater',
3136         'only_matching': True,
3137     }]
3138
3139     def _real_extract(self, url):
3140         return self.url_result(
3141             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
3142
3143
3144 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
3145     IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
3146     _VALID_URL = r':ytrec(?:ommended)?'
3147     _FEED_NAME = 'recommended'
3148     _TESTS = [{
3149         'url': ':ytrec',
3150         'only_matching': True,
3151     }, {
3152         'url': ':ytrecommended',
3153         'only_matching': True,
3154     }]
3155
3156
3157 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
3158     IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
3159     _VALID_URL = r':ytsubs(?:criptions)?'
3160     _FEED_NAME = 'subscriptions'
3161     _TESTS = [{
3162         'url': ':ytsubs',
3163         'only_matching': True,
3164     }, {
3165         'url': ':ytsubscriptions',
3166         'only_matching': True,
3167     }]
3168
3169
3170 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
3171     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
3172     _VALID_URL = r':ythistory'
3173     _FEED_NAME = 'history'
3174     _TESTS = [{
3175         'url': ':ythistory',
3176         'only_matching': True,
3177     }]
3178
3179
3180 class YoutubeTruncatedURLIE(InfoExtractor):
3181     IE_NAME = 'youtube:truncated_url'
3182     IE_DESC = False  # Do not list
3183     _VALID_URL = r'''(?x)
3184         (?:https?://)?
3185         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
3186         (?:watch\?(?:
3187             feature=[a-z_]+|
3188             annotation_id=annotation_[^&]+|
3189             x-yt-cl=[0-9]+|
3190             hl=[^&]*|
3191             t=[0-9]+
3192         )?
3193         |
3194             attribution_link\?a=[^&]+
3195         )
3196         $
3197     '''
3198
3199     _TESTS = [{
3200         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
3201         'only_matching': True,
3202     }, {
3203         'url': 'https://www.youtube.com/watch?',
3204         'only_matching': True,
3205     }, {
3206         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
3207         'only_matching': True,
3208     }, {
3209         'url': 'https://www.youtube.com/watch?feature=foo',
3210         'only_matching': True,
3211     }, {
3212         'url': 'https://www.youtube.com/watch?hl=en-GB',
3213         'only_matching': True,
3214     }, {
3215         'url': 'https://www.youtube.com/watch?t=2372',
3216         'only_matching': True,
3217     }]
3218
3219     def _real_extract(self, url):
3220         raise ExtractorError(
3221             'Did you forget to quote the URL? Remember that & is a meta '
3222             'character in most shells, so you want to put the URL in quotes, '
3223             'like  youtube-dl '
3224             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
3225             ' or simply  youtube-dl BaW_jenozKc  .',
3226             expected=True)
3227
3228
3229 class YoutubeTruncatedIDIE(InfoExtractor):
3230     IE_NAME = 'youtube:truncated_id'
3231     IE_DESC = False  # Do not list
3232     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
3233
3234     _TESTS = [{
3235         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
3236         'only_matching': True,
3237     }]
3238
3239     def _real_extract(self, url):
3240         video_id = self._match_id(url)
3241         raise ExtractorError(
3242             'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
3243             expected=True)