youtube_dl/extractor/youtube.py

   1 # coding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5 import itertools
   6 import json
   7 import os.path
   8 import random
   9 import re
  10 import traceback
  11
  12 from .common import InfoExtractor, SearchInfoExtractor
  13 from ..compat import (
  14     compat_chr,
  15     compat_HTTPError,
  16     compat_parse_qs,
  17     compat_str,
  18     compat_urllib_parse_unquote_plus,
  19     compat_urllib_parse_urlencode,
  20     compat_urllib_parse_urlparse,
  21     compat_urlparse,
  22 )
  23 from ..jsinterp import JSInterpreter
  24 from ..utils import (
  25     ExtractorError,
  26     clean_html,
  27     dict_get,
  28     float_or_none,
  29     int_or_none,
  30     mimetype2ext,
  31     parse_codecs,
  32     parse_duration,
  33     qualities,
  34     remove_start,
  35     smuggle_url,
  36     str_or_none,
  37     str_to_int,
  38     try_get,
  39     unescapeHTML,
  40     unified_strdate,
  41     unsmuggle_url,
  42     update_url_query,
  43     url_or_none,
  44     urlencode_postdata,
  45     urljoin,
  46 )
  47
  48
  49 class YoutubeBaseInfoExtractor(InfoExtractor):
  50     """Provide base functions for Youtube extractors"""
  51     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
  52     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
  53
  54     _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
  55     _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
  56     _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
  57
  58     _NETRC_MACHINE = 'youtube'
  59     # If True it will raise an error if no login info is provided
  60     _LOGIN_REQUIRED = False
  61
  62     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)'
  63
  64     def _ids_to_results(self, ids):
  65         return [
  66             self.url_result(vid_id, 'Youtube', video_id=vid_id)
  67             for vid_id in ids]
  68
  69     def _login(self):
  70         """
  71         Attempt to log in to YouTube.
  72         True is returned if successful or skipped.
  73         False is returned if login failed.
  74
  75         If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
  76         """
  77         username, password = self._get_login_info()
  78         # No authentication to be performed
  79         if username is None:
  80             if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None:
  81                 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
  82             return True
  83
  84         login_page = self._download_webpage(
  85             self._LOGIN_URL, None,
  86             note='Downloading login page',
  87             errnote='unable to fetch login page', fatal=False)
  88         if login_page is False:
  89             return
  90
  91         login_form = self._hidden_inputs(login_page)
  92
  93         def req(url, f_req, note, errnote):
  94             data = login_form.copy()
  95             data.update({
  96                 'pstMsg': 1,
  97                 'checkConnection': 'youtube',
  98                 'checkedDomains': 'youtube',
  99                 'hl': 'en',
 100                 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
 101                 'f.req': json.dumps(f_req),
 102                 'flowName': 'GlifWebSignIn',
 103                 'flowEntry': 'ServiceLogin',
 104                 # TODO: reverse actual botguard identifier generation algo
 105                 'bgRequest': '["identifier",""]',
 106             })
 107             return self._download_json(
 108                 url, None, note=note, errnote=errnote,
 109                 transform_source=lambda s: re.sub(r'^[^[]*', '', s),
 110                 fatal=False,
 111                 data=urlencode_postdata(data), headers={
 112                     'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
 113                     'Google-Accounts-XSRF': 1,
 114                 })
 115
 116         def warn(message):
 117             self._downloader.report_warning(message)
 118
 119         lookup_req = [
 120             username,
 121             None, [], None, 'US', None, None, 2, False, True,
 122             [
 123                 None, None,
 124                 [2, 1, None, 1,
 125                  'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
 126                  None, [], 4],
 127                 1, [None, None, []], None, None, None, True
 128             ],
 129             username,
 130         ]
 131
 132         lookup_results = req(
 133             self._LOOKUP_URL, lookup_req,
 134             'Looking up account info', 'Unable to look up account info')
 135
 136         if lookup_results is False:
 137             return False
 138
 139         user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
 140         if not user_hash:
 141             warn('Unable to extract user hash')
 142             return False
 143
 144         challenge_req = [
 145             user_hash,
 146             None, 1, None, [1, None, None, None, [password, None, True]],
 147             [
 148                 None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
 149                 1, [None, None, []], None, None, None, True
 150             ]]
 151
 152         challenge_results = req(
 153             self._CHALLENGE_URL, challenge_req,
 154             'Logging in', 'Unable to log in')
 155
 156         if challenge_results is False:
 157             return
 158
 159         login_res = try_get(challenge_results, lambda x: x[0][5], list)
 160         if login_res:
 161             login_msg = try_get(login_res, lambda x: x[5], compat_str)
 162             warn(
 163                 'Unable to login: %s' % 'Invalid password'
 164                 if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
 165             return False
 166
 167         res = try_get(challenge_results, lambda x: x[0][-1], list)
 168         if not res:
 169             warn('Unable to extract result entry')
 170             return False
 171
 172         login_challenge = try_get(res, lambda x: x[0][0], list)
 173         if login_challenge:
 174             challenge_str = try_get(login_challenge, lambda x: x[2], compat_str)
 175             if challenge_str == 'TWO_STEP_VERIFICATION':
 176                 # SEND_SUCCESS - TFA code has been successfully sent to phone
 177                 # QUOTA_EXCEEDED - reached the limit of TFA codes
 178                 status = try_get(login_challenge, lambda x: x[5], compat_str)
 179                 if status == 'QUOTA_EXCEEDED':
 180                     warn('Exceeded the limit of TFA codes, try later')
 181                     return False
 182
 183                 tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
 184                 if not tl:
 185                     warn('Unable to extract TL')
 186                     return False
 187
 188                 tfa_code = self._get_tfa_info('2-step verification code')
 189
 190                 if not tfa_code:
 191                     warn(
 192                         'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
 193                         '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
 194                     return False
 195
 196                 tfa_code = remove_start(tfa_code, 'G-')
 197
 198                 tfa_req = [
 199                     user_hash, None, 2, None,
 200                     [
 201                         9, None, None, None, None, None, None, None,
 202                         [None, tfa_code, True, 2]
 203                     ]]
 204
 205                 tfa_results = req(
 206                     self._TFA_URL.format(tl), tfa_req,
 207                     'Submitting TFA code', 'Unable to submit TFA code')
 208
 209                 if tfa_results is False:
 210                     return False
 211
 212                 tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
 213                 if tfa_res:
 214                     tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
 215                     warn(
 216                         'Unable to finish TFA: %s' % 'Invalid TFA code'
 217                         if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
 218                     return False
 219
 220                 check_cookie_url = try_get(
 221                     tfa_results, lambda x: x[0][-1][2], compat_str)
 222             else:
 223                 CHALLENGES = {
 224                     'LOGIN_CHALLENGE': "This device isn't recognized. For your security, Google wants to make sure it's really you.",
 225                     'USERNAME_RECOVERY': 'Please provide additional information to aid in the recovery process.',
 226                     'REAUTH': "There is something unusual about your activity. For your security, Google wants to make sure it's really you.",
 227                 }
 228                 challenge = CHALLENGES.get(
 229                     challenge_str,
 230                     '%s returned error %s.' % (self.IE_NAME, challenge_str))
 231                 warn('%s\nGo to https://accounts.google.com/, login and solve a challenge.' % challenge)
 232                 return False
 233         else:
 234             check_cookie_url = try_get(res, lambda x: x[2], compat_str)
 235
 236         if not check_cookie_url:
 237             warn('Unable to extract CheckCookie URL')
 238             return False
 239
 240         check_cookie_results = self._download_webpage(
 241             check_cookie_url, None, 'Checking cookie', fatal=False)
 242
 243         if check_cookie_results is False:
 244             return False
 245
 246         if 'https://myaccount.google.com/' not in check_cookie_results:
 247             warn('Unable to log in')
 248             return False
 249
 250         return True
 251
 252     def _initialize_consent(self):
 253         cookies = self._get_cookies('https://www.youtube.com/')
 254         if cookies.get('__Secure-3PSID'):
 255             return
 256         consent_id = None
 257         consent = cookies.get('CONSENT')
 258         if consent:
 259             if 'YES' in consent.value:
 260                 return
 261             consent_id = self._search_regex(
 262                 r'PENDING\+(\d+)', consent.value, 'consent', default=None)
 263         if not consent_id:
 264             consent_id = random.randint(100, 999)
 265         self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 266
 267     def _real_initialize(self):
 268         self._initialize_consent()
 269         if self._downloader is None:
 270             return
 271         if not self._login():
 272             return
 273
 274     _DEFAULT_API_DATA = {
 275         'context': {
 276             'client': {
 277                 'clientName': 'WEB',
 278                 'clientVersion': '2.20201021.03.00',
 279             }
 280         },
 281     }
 282
 283     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
 284     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
 285     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 286
 287     def _call_api(self, ep, query, video_id, fatal=True):
 288         data = self._DEFAULT_API_DATA.copy()
 289         data.update(query)
 290
 291         return self._download_json(
 292             'https://www.youtube.com/youtubei/v1/%s' % ep, video_id=video_id,
 293             note='Downloading API JSON', errnote='Unable to download API page',
 294             data=json.dumps(data).encode('utf8'), fatal=fatal,
 295             headers={'content-type': 'application/json'},
 296             query={'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'})
 297
 298     def _extract_yt_initial_data(self, video_id, webpage):
 299         return self._parse_json(
 300             self._search_regex(
 301                 (r'%s\s*%s' % (self._YT_INITIAL_DATA_RE, self._YT_INITIAL_BOUNDARY_RE),
 302                  self._YT_INITIAL_DATA_RE), webpage, 'yt initial data'),
 303             video_id)
 304
 305     def _extract_ytcfg(self, video_id, webpage):
 306         return self._parse_json(
 307             self._search_regex(
 308                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
 309                 default='{}'), video_id, fatal=False)
 310
 311     def _extract_video(self, renderer):
 312         video_id = renderer['videoId']
 313         title = try_get(
 314             renderer,
 315             (lambda x: x['title']['runs'][0]['text'],
 316              lambda x: x['title']['simpleText']), compat_str)
 317         description = try_get(
 318             renderer, lambda x: x['descriptionSnippet']['runs'][0]['text'],
 319             compat_str)
 320         duration = parse_duration(try_get(
 321             renderer, lambda x: x['lengthText']['simpleText'], compat_str))
 322         view_count_text = try_get(
 323             renderer, lambda x: x['viewCountText']['simpleText'], compat_str) or ''
 324         view_count = str_to_int(self._search_regex(
 325             r'^([\d,]+)', re.sub(r'\s', '', view_count_text),
 326             'view count', default=None))
 327         uploader = try_get(
 328             renderer,
 329             (lambda x: x['ownerText']['runs'][0]['text'],
 330              lambda x: x['shortBylineText']['runs'][0]['text']), compat_str)
 331         return {
 332             '_type': 'url',
 333             'ie_key': YoutubeIE.ie_key(),
 334             'id': video_id,
 335             'url': video_id,
 336             'title': title,
 337             'description': description,
 338             'duration': duration,
 339             'view_count': view_count,
 340             'uploader': uploader,
 341         }
 342
 343
 344 class YoutubeIE(YoutubeBaseInfoExtractor):
 345     IE_DESC = 'YouTube.com'
 346     _INVIDIOUS_SITES = (
 347         # invidious-redirect websites
 348         r'(?:www\.)?redirect\.invidious\.io',
 349         r'(?:(?:www|dev)\.)?invidio\.us',
 350         # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
 351         r'(?:(?:www|no)\.)?invidiou\.sh',
 352         r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
 353         r'(?:www\.)?invidious\.kabi\.tk',
 354         r'(?:www\.)?invidious\.13ad\.de',
 355         r'(?:www\.)?invidious\.mastodon\.host',
 356         r'(?:www\.)?invidious\.zapashcanon\.fr',
 357         r'(?:www\.)?invidious\.kavin\.rocks',
 358         r'(?:www\.)?invidious\.tube',
 359         r'(?:www\.)?invidiou\.site',
 360         r'(?:www\.)?invidious\.site',
 361         r'(?:www\.)?invidious\.xyz',
 362         r'(?:www\.)?invidious\.nixnet\.xyz',
 363         r'(?:www\.)?invidious\.drycat\.fr',
 364         r'(?:www\.)?tube\.poal\.co',
 365         r'(?:www\.)?tube\.connect\.cafe',
 366         r'(?:www\.)?vid\.wxzm\.sx',
 367         r'(?:www\.)?vid\.mint\.lgbt',
 368         r'(?:www\.)?yewtu\.be',
 369         r'(?:www\.)?yt\.elukerio\.org',
 370         r'(?:www\.)?yt\.lelux\.fi',
 371         r'(?:www\.)?invidious\.ggc-project\.de',
 372         r'(?:www\.)?yt\.maisputain\.ovh',
 373         r'(?:www\.)?invidious\.13ad\.de',
 374         r'(?:www\.)?invidious\.toot\.koeln',
 375         r'(?:www\.)?invidious\.fdn\.fr',
 376         r'(?:www\.)?watch\.nettohikari\.com',
 377         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
 378         r'(?:www\.)?qklhadlycap4cnod\.onion',
 379         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
 380         r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
 381         r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
 382         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
 383         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
 384         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
 385     )
 386     _VALID_URL = r"""(?x)^
 387                      (
 388                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 389                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
 390                             (?:www\.)?deturl\.com/www\.youtube\.com|
 391                             (?:www\.)?pwnyoutube\.com|
 392                             (?:www\.)?hooktube\.com|
 393                             (?:www\.)?yourepeat\.com|
 394                             tube\.majestyc\.net|
 395                             %(invidious)s|
 396                             youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
 397                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 398                          (?:                                                  # the various things that can precede the ID:
 399                              (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
 400                              |(?:                                             # or the v= param in all its forms
 401                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 402                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 403                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 404                                  v=
 405                              )
 406                          ))
 407                          |(?:
 408                             youtu\.be|                                        # just youtu.be/xxxx
 409                             vid\.plus|                                        # or vid.plus/xxxx
 410                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 411                             %(invidious)s
 412                          )/
 413                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 414                          )
 415                      )?                                                       # all until now is optional -> you can pass the naked ID
 416                      (?P<id>[0-9A-Za-z_-]{11})                                      # here is it! the YouTube video ID
 417                      (?!.*?\blist=
 418                         (?:
 419                             %(playlist_id)s|                                  # combined list/video URLs are handled by the playlist IE
 420                             WL                                                # WL are handled by the watch later IE
 421                         )
 422                      )
 423                      (?(1).+)?                                                # if we found the ID, everything can follow
 424                      $""" % {
 425         'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
 426         'invidious': '|'.join(_INVIDIOUS_SITES),
 427     }
 428     _PLAYER_INFO_RE = (
 429         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
 430         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
 431         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
 432     )
 433     _SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt')
 434
 435     _GEO_BYPASS = False
 436
 437     IE_NAME = 'youtube'
 438     _TESTS = [
 439         {
 440             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
 441             'info_dict': {
 442                 'id': 'BaW_jenozKc',
 443                 'ext': 'mp4',
 444                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
 445                 'uploader': 'Philipp Hagemeister',
 446                 'uploader_id': 'phihag',
 447                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
 448                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
 449                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
 450                 'upload_date': '20121002',
 451                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
 452                 'categories': ['Science & Technology'],
 453                 'tags': ['youtube-dl'],
 454                 'duration': 10,
 455                 'view_count': int,
 456                 'like_count': int,
 457                 'dislike_count': int,
 458                 'start_time': 1,
 459                 'end_time': 9,
 460             }
 461         },
 462         {
 463             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
 464             'note': 'Embed-only video (#1746)',
 465             'info_dict': {
 466                 'id': 'yZIXLfi8CZQ',
 467                 'ext': 'mp4',
 468                 'upload_date': '20120608',
 469                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
 470                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
 471                 'uploader': 'SET India',
 472                 'uploader_id': 'setindia',
 473                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
 474                 'age_limit': 18,
 475             },
 476             'skip': 'Private video',
 477         },
 478         {
 479             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
 480             'note': 'Use the first video ID in the URL',
 481             'info_dict': {
 482                 'id': 'BaW_jenozKc',
 483                 'ext': 'mp4',
 484                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
 485                 'uploader': 'Philipp Hagemeister',
 486                 'uploader_id': 'phihag',
 487                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
 488                 'upload_date': '20121002',
 489                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
 490                 'categories': ['Science & Technology'],
 491                 'tags': ['youtube-dl'],
 492                 'duration': 10,
 493                 'view_count': int,
 494                 'like_count': int,
 495                 'dislike_count': int,
 496             },
 497             'params': {
 498                 'skip_download': True,
 499             },
 500         },
 501         {
 502             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
 503             'note': '256k DASH audio (format 141) via DASH manifest',
 504             'info_dict': {
 505                 'id': 'a9LDPn-MO4I',
 506                 'ext': 'm4a',
 507                 'upload_date': '20121002',
 508                 'uploader_id': '8KVIDEO',
 509                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
 510                 'description': '',
 511                 'uploader': '8KVIDEO',
 512                 'title': 'UHDTV TEST 8K VIDEO.mp4'
 513             },
 514             'params': {
 515                 'youtube_include_dash_manifest': True,
 516                 'format': '141',
 517             },
 518             'skip': 'format 141 not served anymore',
 519         },
 520         # DASH manifest with encrypted signature
 521         {
 522             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
 523             'info_dict': {
 524                 'id': 'IB3lcPjvWLA',
 525                 'ext': 'm4a',
 526                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
 527                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
 528                 'duration': 244,
 529                 'uploader': 'AfrojackVEVO',
 530                 'uploader_id': 'AfrojackVEVO',
 531                 'upload_date': '20131011',
 532                 'abr': 129.495,
 533             },
 534             'params': {
 535                 'youtube_include_dash_manifest': True,
 536                 'format': '141/bestaudio[ext=m4a]',
 537             },
 538         },
 539         # Controversy video
 540         {
 541             'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
 542             'info_dict': {
 543                 'id': 'T4XJQO3qol8',
 544                 'ext': 'mp4',
 545                 'duration': 219,
 546                 'upload_date': '20100909',
 547                 'uploader': 'Amazing Atheist',
 548                 'uploader_id': 'TheAmazingAtheist',
 549                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
 550                 'title': 'Burning Everyone\'s Koran',
 551                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms \r\n\r\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
 552             }
 553         },
 554         # Normal age-gate video (No vevo, embed allowed), available via embed page
 555         {
 556             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
 557             'info_dict': {
 558                 'id': 'HtVdAasjOgU',
 559                 'ext': 'mp4',
 560                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
 561                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
 562                 'duration': 142,
 563                 'uploader': 'The Witcher',
 564                 'uploader_id': 'WitcherGame',
 565                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
 566                 'upload_date': '20140605',
 567                 'age_limit': 18,
 568             },
 569         },
 570         {
 571             # Age-gated video only available with authentication (unavailable
 572             # via embed page workaround)
 573             'url': 'XgnwCQzjau8',
 574             'only_matching': True,
 575         },
 576         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
 577         # YouTube Red ad is not captured for creator
 578         {
 579             'url': '__2ABJjxzNo',
 580             'info_dict': {
 581                 'id': '__2ABJjxzNo',
 582                 'ext': 'mp4',
 583                 'duration': 266,
 584                 'upload_date': '20100430',
 585                 'uploader_id': 'deadmau5',
 586                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
 587                 'creator': 'deadmau5',
 588                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
 589                 'uploader': 'deadmau5',
 590                 'title': 'Deadmau5 - Some Chords (HD)',
 591                 'alt_title': 'Some Chords',
 592             },
 593             'expected_warnings': [
 594                 'DASH manifest missing',
 595             ]
 596         },
 597         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
 598         {
 599             'url': 'lqQg6PlCWgI',
 600             'info_dict': {
 601                 'id': 'lqQg6PlCWgI',
 602                 'ext': 'mp4',
 603                 'duration': 6085,
 604                 'upload_date': '20150827',
 605                 'uploader_id': 'olympic',
 606                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
 607                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
 608                 'uploader': 'Olympic',
 609                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
 610             },
 611             'params': {
 612                 'skip_download': 'requires avconv',
 613             }
 614         },
 615         # Non-square pixels
 616         {
 617             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
 618             'info_dict': {
 619                 'id': '_b-2C3KPAM0',
 620                 'ext': 'mp4',
 621                 'stretched_ratio': 16 / 9.,
 622                 'duration': 85,
 623                 'upload_date': '20110310',
 624                 'uploader_id': 'AllenMeow',
 625                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
 626                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
 627                 'uploader': '孫ᄋᄅ',
 628                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
 629             },
 630         },
 631         # url_encoded_fmt_stream_map is empty string
 632         {
 633             'url': 'qEJwOuvDf7I',
 634             'info_dict': {
 635                 'id': 'qEJwOuvDf7I',
 636                 'ext': 'webm',
 637                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
 638                 'description': '',
 639                 'upload_date': '20150404',
 640                 'uploader_id': 'spbelect',
 641                 'uploader': 'Наблюдатели Петербурга',
 642             },
 643             'params': {
 644                 'skip_download': 'requires avconv',
 645             },
 646             'skip': 'This live event has ended.',
 647         },
 648         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
 649         {
 650             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
 651             'info_dict': {
 652                 'id': 'FIl7x6_3R5Y',
 653                 'ext': 'webm',
 654                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
 655                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
 656                 'duration': 220,
 657                 'upload_date': '20150625',
 658                 'uploader_id': 'dorappi2000',
 659                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
 660                 'uploader': 'dorappi2000',
 661                 'formats': 'mincount:31',
 662             },
 663             'skip': 'not actual anymore',
 664         },
 665         # DASH manifest with segment_list
 666         {
 667             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
 668             'md5': '8ce563a1d667b599d21064e982ab9e31',
 669             'info_dict': {
 670                 'id': 'CsmdDsKjzN8',
 671                 'ext': 'mp4',
 672                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
 673                 'uploader': 'Airtek',
 674                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
 675                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
 676                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
 677             },
 678             'params': {
 679                 'youtube_include_dash_manifest': True,
 680                 'format': '135',  # bestvideo
 681             },
 682             'skip': 'This live event has ended.',
 683         },
 684         {
 685             # Multifeed videos (multiple cameras), URL is for Main Camera
 686             'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
 687             'info_dict': {
 688                 'id': 'jvGDaLqkpTg',
 689                 'title': 'Tom Clancy Free Weekend Rainbow Whatever',
 690                 'description': 'md5:e03b909557865076822aa169218d6a5d',
 691             },
 692             'playlist': [{
 693                 'info_dict': {
 694                     'id': 'jvGDaLqkpTg',
 695                     'ext': 'mp4',
 696                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
 697                     'description': 'md5:e03b909557865076822aa169218d6a5d',
 698                     'duration': 10643,
 699                     'upload_date': '20161111',
 700                     'uploader': 'Team PGP',
 701                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
 702                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
 703                 },
 704             }, {
 705                 'info_dict': {
 706                     'id': '3AKt1R1aDnw',
 707                     'ext': 'mp4',
 708                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
 709                     'description': 'md5:e03b909557865076822aa169218d6a5d',
 710                     'duration': 10991,
 711                     'upload_date': '20161111',
 712                     'uploader': 'Team PGP',
 713                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
 714                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
 715                 },
 716             }, {
 717                 'info_dict': {
 718                     'id': 'RtAMM00gpVc',
 719                     'ext': 'mp4',
 720                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
 721                     'description': 'md5:e03b909557865076822aa169218d6a5d',
 722                     'duration': 10995,
 723                     'upload_date': '20161111',
 724                     'uploader': 'Team PGP',
 725                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
 726                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
 727                 },
 728             }, {
 729                 'info_dict': {
 730                     'id': '6N2fdlP3C5U',
 731                     'ext': 'mp4',
 732                     'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
 733                     'description': 'md5:e03b909557865076822aa169218d6a5d',
 734                     'duration': 10990,
 735                     'upload_date': '20161111',
 736                     'uploader': 'Team PGP',
 737                     'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
 738                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
 739                 },
 740             }],
 741             'params': {
 742                 'skip_download': True,
 743             },
 744         },
 745         {
 746             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
 747             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
 748             'info_dict': {
 749                 'id': 'gVfLd0zydlo',
 750                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
 751             },
 752             'playlist_count': 2,
 753             'skip': 'Not multifeed anymore',
 754         },
 755         {
 756             'url': 'https://vid.plus/FlRa-iH7PGw',
 757             'only_matching': True,
 758         },
 759         {
 760             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
 761             'only_matching': True,
 762         },
 763         {
 764             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
 765             # Also tests cut-off URL expansion in video description (see
 766             # https://github.com/ytdl-org/youtube-dl/issues/1892,
 767             # https://github.com/ytdl-org/youtube-dl/issues/8164)
 768             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
 769             'info_dict': {
 770                 'id': 'lsguqyKfVQg',
 771                 'ext': 'mp4',
 772                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
 773                 'alt_title': 'Dark Walk - Position Music',
 774                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
 775                 'duration': 133,
 776                 'upload_date': '20151119',
 777                 'uploader_id': 'IronSoulElf',
 778                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
 779                 'uploader': 'IronSoulElf',
 780                 'creator': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
 781                 'track': 'Dark Walk - Position Music',
 782                 'artist': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
 783                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
 784             },
 785             'params': {
 786                 'skip_download': True,
 787             },
 788         },
 789         {
 790             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
 791             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
 792             'only_matching': True,
 793         },
 794         {
 795             # Video with yt:stretch=17:0
 796             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
 797             'info_dict': {
 798                 'id': 'Q39EVAstoRM',
 799                 'ext': 'mp4',
 800                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
 801                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
 802                 'upload_date': '20151107',
 803                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
 804                 'uploader': 'CH GAMER DROID',
 805             },
 806             'params': {
 807                 'skip_download': True,
 808             },
 809             'skip': 'This video does not exist.',
 810         },
 811         {
 812             # Video licensed under Creative Commons
 813             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
 814             'info_dict': {
 815                 'id': 'M4gD1WSo5mA',
 816                 'ext': 'mp4',
 817                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
 818                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
 819                 'duration': 721,
 820                 'upload_date': '20150127',
 821                 'uploader_id': 'BerkmanCenter',
 822                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
 823                 'uploader': 'The Berkman Klein Center for Internet & Society',
 824                 'license': 'Creative Commons Attribution license (reuse allowed)',
 825             },
 826             'params': {
 827                 'skip_download': True,
 828             },
 829         },
 830         {
 831             # Channel-like uploader_url
 832             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
 833             'info_dict': {
 834                 'id': 'eQcmzGIKrzg',
 835                 'ext': 'mp4',
 836                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
 837                 'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
 838                 'duration': 4060,
 839                 'upload_date': '20151119',
 840                 'uploader': 'Bernie Sanders',
 841                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
 842                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
 843                 'license': 'Creative Commons Attribution license (reuse allowed)',
 844             },
 845             'params': {
 846                 'skip_download': True,
 847             },
 848         },
 849         {
 850             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
 851             'only_matching': True,
 852         },
 853         {
 854             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
 855             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
 856             'only_matching': True,
 857         },
 858         {
 859             # Rental video preview
 860             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
 861             'info_dict': {
 862                 'id': 'uGpuVWrhIzE',
 863                 'ext': 'mp4',
 864                 'title': 'Piku - Trailer',
 865                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
 866                 'upload_date': '20150811',
 867                 'uploader': 'FlixMatrix',
 868                 'uploader_id': 'FlixMatrixKaravan',
 869                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
 870                 'license': 'Standard YouTube License',
 871             },
 872             'params': {
 873                 'skip_download': True,
 874             },
 875             'skip': 'This video is not available.',
 876         },
 877         {
 878             # YouTube Red video with episode data
 879             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
 880             'info_dict': {
 881                 'id': 'iqKdEhx-dD4',
 882                 'ext': 'mp4',
 883                 'title': 'Isolation - Mind Field (Ep 1)',
 884                 'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
 885                 'duration': 2085,
 886                 'upload_date': '20170118',
 887                 'uploader': 'Vsauce',
 888                 'uploader_id': 'Vsauce',
 889                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
 890                 'series': 'Mind Field',
 891                 'season_number': 1,
 892                 'episode_number': 1,
 893             },
 894             'params': {
 895                 'skip_download': True,
 896             },
 897             'expected_warnings': [
 898                 'Skipping DASH manifest',
 899             ],
 900         },
 901         {
 902             # The following content has been identified by the YouTube community
 903             # as inappropriate or offensive to some audiences.
 904             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
 905             'info_dict': {
 906                 'id': '6SJNVb0GnPI',
 907                 'ext': 'mp4',
 908                 'title': 'Race Differences in Intelligence',
 909                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
 910                 'duration': 965,
 911                 'upload_date': '20140124',
 912                 'uploader': 'New Century Foundation',
 913                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
 914                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
 915             },
 916             'params': {
 917                 'skip_download': True,
 918             },
 919             'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
 920         },
 921         {
 922             # itag 212
 923             'url': '1t24XAntNCY',
 924             'only_matching': True,
 925         },
 926         {
 927             # geo restricted to JP
 928             'url': 'sJL6WA-aGkQ',
 929             'only_matching': True,
 930         },
 931         {
 932             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
 933             'only_matching': True,
 934         },
 935         {
 936             'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
 937             'only_matching': True,
 938         },
 939         {
 940             # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
 941             'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
 942             'only_matching': True,
 943         },
 944         {
 945             # DRM protected
 946             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
 947             'only_matching': True,
 948         },
 949         {
 950             # Video with unsupported adaptive stream type formats
 951             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
 952             'info_dict': {
 953                 'id': 'Z4Vy8R84T1U',
 954                 'ext': 'mp4',
 955                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
 956                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
 957                 'duration': 433,
 958                 'upload_date': '20130923',
 959                 'uploader': 'Amelia Putri Harwita',
 960                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
 961                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
 962                 'formats': 'maxcount:10',
 963             },
 964             'params': {
 965                 'skip_download': True,
 966                 'youtube_include_dash_manifest': False,
 967             },
 968             'skip': 'not actual anymore',
 969         },
 970         {
 971             # Youtube Music Auto-generated description
 972             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
 973             'info_dict': {
 974                 'id': 'MgNrAu2pzNs',
 975                 'ext': 'mp4',
 976                 'title': 'Voyeur Girl',
 977                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
 978                 'upload_date': '20190312',
 979                 'uploader': 'Stephen - Topic',
 980                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
 981                 'artist': 'Stephen',
 982                 'track': 'Voyeur Girl',
 983                 'album': 'it\'s too much love to know my dear',
 984                 'release_date': '20190313',
 985                 'release_year': 2019,
 986             },
 987             'params': {
 988                 'skip_download': True,
 989             },
 990         },
 991         {
 992             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
 993             'only_matching': True,
 994         },
 995         {
 996             # invalid -> valid video id redirection
 997             'url': 'DJztXj2GPfl',
 998             'info_dict': {
 999                 'id': 'DJztXj2GPfk',
1000                 'ext': 'mp4',
1001                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1002                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1003                 'upload_date': '20090125',
1004                 'uploader': 'Prochorowka',
1005                 'uploader_id': 'Prochorowka',
1006                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1007                 'artist': 'Panjabi MC',
1008                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1009                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1010             },
1011             'params': {
1012                 'skip_download': True,
1013             },
1014             'skip': 'Video unavailable',
1015         },
1016         {
1017             # empty description results in an empty string
1018             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1019             'info_dict': {
1020                 'id': 'x41yOUIvK2k',
1021                 'ext': 'mp4',
1022                 'title': 'IMG 3456',
1023                 'description': '',
1024                 'upload_date': '20170613',
1025                 'uploader_id': 'ElevageOrVert',
1026                 'uploader': 'ElevageOrVert',
1027             },
1028             'params': {
1029                 'skip_download': True,
1030             },
1031         },
1032         {
1033             # with '};' inside yt initial data (see [1])
1034             # see [2] for an example with '};' inside ytInitialPlayerResponse
1035             # 1. https://github.com/ytdl-org/youtube-dl/issues/27093
1036             # 2. https://github.com/ytdl-org/youtube-dl/issues/27216
1037             'url': 'https://www.youtube.com/watch?v=CHqg6qOn4no',
1038             'info_dict': {
1039                 'id': 'CHqg6qOn4no',
1040                 'ext': 'mp4',
1041                 'title': 'Part 77   Sort a list of simple types in c#',
1042                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
1043                 'upload_date': '20130831',
1044                 'uploader_id': 'kudvenkat',
1045                 'uploader': 'kudvenkat',
1046             },
1047             'params': {
1048                 'skip_download': True,
1049             },
1050         },
1051         {
1052             # another example of '};' in ytInitialData
1053             'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
1054             'only_matching': True,
1055         },
1056         {
1057             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
1058             'only_matching': True,
1059         },
1060         {
1061             # https://github.com/ytdl-org/youtube-dl/pull/28094
1062             'url': 'OtqTfy26tG0',
1063             'info_dict': {
1064                 'id': 'OtqTfy26tG0',
1065                 'ext': 'mp4',
1066                 'title': 'Burn Out',
1067                 'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
1068                 'upload_date': '20141120',
1069                 'uploader': 'The Cinematic Orchestra - Topic',
1070                 'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
1071                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
1072                 'artist': 'The Cinematic Orchestra',
1073                 'track': 'Burn Out',
1074                 'album': 'Every Day',
1075                 'release_data': None,
1076                 'release_year': None,
1077             },
1078             'params': {
1079                 'skip_download': True,
1080             },
1081         },
1082         {
1083             # controversial video, only works with bpctr when authenticated with cookies
1084             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
1085             'only_matching': True,
1086         },
1087         {
1088             # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
1089             'url': 'cBvYw8_A0vQ',
1090             'info_dict': {
1091                 'id': 'cBvYw8_A0vQ',
1092                 'ext': 'mp4',
1093                 'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
1094                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
1095                 'upload_date': '20201120',
1096                 'uploader': 'Walk around Japan',
1097                 'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
1098                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
1099             },
1100             'params': {
1101                 'skip_download': True,
1102             },
1103         },
1104     ]
1105     _formats = {
1106         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1107         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
1108         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
1109         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
1110         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
1111         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1112         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1113         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1114         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
1115         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
1116         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1117         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
1118         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1119         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
1120         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1121         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
1122         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1123         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
1124
1125
1126         # 3D videos
1127         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1128         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
1129         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1130         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
1131         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
1132         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1133         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
1134
1135         # Apple HTTP Live Streaming
1136         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1137         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1138         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1139         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
1140         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1141         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
1142         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
1143         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
1144
1145         # DASH mp4 video
1146         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
1147         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
1148         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1149         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
1150         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
1151         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
1152         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
1153         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
1154         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
1155         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1156         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
1157         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
1158
1159         # Dash mp4 audio
1160         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
1161         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
1162         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
1163         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1164         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
1165         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
1166         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
1167
1168         # Dash webm
1169         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1170         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1171         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1172         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1173         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1174         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
1175         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
1176         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1177         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1178         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1179         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1180         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1181         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1182         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1183         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1184         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
1185         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1186         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1187         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1188         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1189         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
1190         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
1191
1192         # Dash webm audio
1193         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
1194         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
1195
1196         # Dash webm audio with opus inside
1197         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
1198         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
1199         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
1200
1201         # RTMP (unnamed)
1202         '_rtmp': {'protocol': 'rtmp'},
1203
1204         # av01 video only formats sometimes served with "unknown" codecs
1205         '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1206         '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1207         '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1208         '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
1209     }
1210
1211     def __init__(self, *args, **kwargs):
1212         super(YoutubeIE, self).__init__(*args, **kwargs)
1213         self._code_cache = {}
1214         self._player_cache = {}
1215
1216     def _signature_cache_id(self, example_sig):
1217         """ Return a string representation of a signature """
1218         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
1219
1220     @classmethod
1221     def _extract_player_info(cls, player_url):
1222         for player_re in cls._PLAYER_INFO_RE:
1223             id_m = re.search(player_re, player_url)
1224             if id_m:
1225                 break
1226         else:
1227             raise ExtractorError('Cannot identify player %r' % player_url)
1228         return id_m.group('id')
1229
1230     def _extract_signature_function(self, video_id, player_url, example_sig):
1231         player_id = self._extract_player_info(player_url)
1232
1233         # Read from filesystem cache
1234         func_id = 'js_%s_%s' % (
1235             player_id, self._signature_cache_id(example_sig))
1236         assert os.path.basename(func_id) == func_id
1237
1238         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
1239         if cache_spec is not None:
1240             return lambda s: ''.join(s[i] for i in cache_spec)
1241
1242         if player_id not in self._code_cache:
1243             self._code_cache[player_id] = self._download_webpage(
1244                 player_url, video_id,
1245                 note='Downloading player ' + player_id,
1246                 errnote='Download of %s failed' % player_url)
1247         code = self._code_cache[player_id]
1248         res = self._parse_sig_js(code)
1249
1250         test_string = ''.join(map(compat_chr, range(len(example_sig))))
1251         cache_res = res(test_string)
1252         cache_spec = [ord(c) for c in cache_res]
1253
1254         self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
1255         return res
1256
1257     def _print_sig_code(self, func, example_sig):
1258         def gen_sig_code(idxs):
1259             def _genslice(start, end, step):
1260                 starts = '' if start == 0 else str(start)
1261                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
1262                 steps = '' if step == 1 else (':%d' % step)
1263                 return 's[%s%s%s]' % (starts, ends, steps)
1264
1265             step = None
1266             # Quelch pyflakes warnings - start will be set when step is set
1267             start = '(Never used)'
1268             for i, prev in zip(idxs[1:], idxs[:-1]):
1269                 if step is not None:
1270                     if i - prev == step:
1271                         continue
1272                     yield _genslice(start, prev, step)
1273                     step = None
1274                     continue
1275                 if i - prev in [-1, 1]:
1276                     step = i - prev
1277                     start = prev
1278                     continue
1279                 else:
1280                     yield 's[%d]' % prev
1281             if step is None:
1282                 yield 's[%d]' % i
1283             else:
1284                 yield _genslice(start, i, step)
1285
1286         test_string = ''.join(map(compat_chr, range(len(example_sig))))
1287         cache_res = func(test_string)
1288         cache_spec = [ord(c) for c in cache_res]
1289         expr_code = ' + '.join(gen_sig_code(cache_spec))
1290         signature_id_tuple = '(%s)' % (
1291             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
1292         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
1293                 '    return %s\n') % (signature_id_tuple, expr_code)
1294         self.to_screen('Extracted signature function:\n' + code)
1295
1296     def _parse_sig_js(self, jscode):
1297         funcname = self._search_regex(
1298             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1299              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1300              r'\bm=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(h\.s\)\)',
1301              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(c\)\)',
1302              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
1303              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1304              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1305              # Obsolete patterns
1306              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1307              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
1308              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1309              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1310              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1311              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1312              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1313              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
1314             jscode, 'Initial JS player signature function name', group='sig')
1315
1316         jsi = JSInterpreter(jscode)
1317         initial_function = jsi.extract_function(funcname)
1318         return lambda s: initial_function([s])
1319
1320     def _decrypt_signature(self, s, video_id, player_url):
1321         """Turn the encrypted s field into a working signature"""
1322
1323         if player_url is None:
1324             raise ExtractorError('Cannot decrypt signature without player_url')
1325
1326         if player_url.startswith('//'):
1327             player_url = 'https:' + player_url
1328         elif not re.match(r'https?://', player_url):
1329             player_url = compat_urlparse.urljoin(
1330                 'https://www.youtube.com', player_url)
1331         try:
1332             player_id = (player_url, self._signature_cache_id(s))
1333             if player_id not in self._player_cache:
1334                 func = self._extract_signature_function(
1335                     video_id, player_url, s
1336                 )
1337                 self._player_cache[player_id] = func
1338             func = self._player_cache[player_id]
1339             if self._downloader.params.get('youtube_print_sig_code'):
1340                 self._print_sig_code(func, s)
1341             return func(s)
1342         except Exception as e:
1343             tb = traceback.format_exc()
1344             raise ExtractorError(
1345                 'Signature extraction failed: ' + tb, cause=e)
1346
1347     def _mark_watched(self, video_id, player_response):
1348         playback_url = url_or_none(try_get(
1349             player_response,
1350             lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']))
1351         if not playback_url:
1352             return
1353         parsed_playback_url = compat_urlparse.urlparse(playback_url)
1354         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
1355
1356         # cpn generation algorithm is reverse engineered from base.js.
1357         # In fact it works even with dummy cpn.
1358         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
1359         cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
1360
1361         qs.update({
1362             'ver': ['2'],
1363             'cpn': [cpn],
1364         })
1365         playback_url = compat_urlparse.urlunparse(
1366             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
1367
1368         self._download_webpage(
1369             playback_url, video_id, 'Marking watched',
1370             'Unable to mark watched', fatal=False)
1371
1372     @staticmethod
1373     def _extract_urls(webpage):
1374         # Embedded YouTube player
1375         entries = [
1376             unescapeHTML(mobj.group('url'))
1377             for mobj in re.finditer(r'''(?x)
1378             (?:
1379                 <iframe[^>]+?src=|
1380                 data-video-url=|
1381                 <embed[^>]+?src=|
1382                 embedSWF\(?:\s*|
1383                 <object[^>]+data=|
1384                 new\s+SWFObject\(
1385             )
1386             (["\'])
1387                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1388                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1389             \1''', webpage)]
1390
1391         # lazyYT YouTube embed
1392         entries.extend(list(map(
1393             unescapeHTML,
1394             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
1395
1396         # Wordpress "YouTube Video Importer" plugin
1397         matches = re.findall(r'''(?x)<div[^>]+
1398             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
1399             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
1400         entries.extend(m[-1] for m in matches)
1401
1402         return entries
1403
1404     @staticmethod
1405     def _extract_url(webpage):
1406         urls = YoutubeIE._extract_urls(webpage)
1407         return urls[0] if urls else None
1408
1409     @classmethod
1410     def extract_id(cls, url):
1411         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
1412         if mobj is None:
1413             raise ExtractorError('Invalid URL: %s' % url)
1414         video_id = mobj.group(2)
1415         return video_id
1416
1417     def _extract_chapters_from_json(self, data, video_id, duration):
1418         chapters_list = try_get(
1419             data,
1420             lambda x: x['playerOverlays']
1421                        ['playerOverlayRenderer']
1422                        ['decoratedPlayerBarRenderer']
1423                        ['decoratedPlayerBarRenderer']
1424                        ['playerBar']
1425                        ['chapteredPlayerBarRenderer']
1426                        ['chapters'],
1427             list)
1428         if not chapters_list:
1429             return
1430
1431         def chapter_time(chapter):
1432             return float_or_none(
1433                 try_get(
1434                     chapter,
1435                     lambda x: x['chapterRenderer']['timeRangeStartMillis'],
1436                     int),
1437                 scale=1000)
1438         chapters = []
1439         for next_num, chapter in enumerate(chapters_list, start=1):
1440             start_time = chapter_time(chapter)
1441             if start_time is None:
1442                 continue
1443             end_time = (chapter_time(chapters_list[next_num])
1444                         if next_num < len(chapters_list) else duration)
1445             if end_time is None:
1446                 continue
1447             title = try_get(
1448                 chapter, lambda x: x['chapterRenderer']['title']['simpleText'],
1449                 compat_str)
1450             chapters.append({
1451                 'start_time': start_time,
1452                 'end_time': end_time,
1453                 'title': title,
1454             })
1455         return chapters
1456
1457     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
1458         return self._parse_json(self._search_regex(
1459             (r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE),
1460              regex), webpage, name, default='{}'), video_id, fatal=False)
1461
1462     def _real_extract(self, url):
1463         url, smuggled_data = unsmuggle_url(url, {})
1464         video_id = self._match_id(url)
1465         base_url = self.http_scheme() + '//www.youtube.com/'
1466         webpage_url = base_url + 'watch?v=' + video_id
1467         webpage = self._download_webpage(
1468             webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
1469
1470         player_response = None
1471         if webpage:
1472             player_response = self._extract_yt_initial_variable(
1473                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
1474                 video_id, 'initial player response')
1475         if not player_response:
1476             player_response = self._call_api(
1477                 'player', {'videoId': video_id}, video_id)
1478
1479         playability_status = player_response.get('playabilityStatus') or {}
1480         if playability_status.get('reason') == 'Sign in to confirm your age':
1481             pr = self._parse_json(try_get(compat_parse_qs(
1482                 self._download_webpage(
1483                     base_url + 'get_video_info', video_id,
1484                     'Refetching age-gated info webpage',
1485                     'unable to download video info webpage', query={
1486                         'video_id': video_id,
1487                         'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1488                     }, fatal=False)),
1489                 lambda x: x['player_response'][0],
1490                 compat_str) or '{}', video_id)
1491             if pr:
1492                 player_response = pr
1493
1494         trailer_video_id = try_get(
1495             playability_status,
1496             lambda x: x['errorScreen']['playerLegacyDesktopYpcTrailerRenderer']['trailerVideoId'],
1497             compat_str)
1498         if trailer_video_id:
1499             return self.url_result(
1500                 trailer_video_id, self.ie_key(), trailer_video_id)
1501
1502         def get_text(x):
1503             if not x:
1504                 return
1505             text = x.get('simpleText')
1506             if text and isinstance(text, compat_str):
1507                 return text
1508             runs = x.get('runs')
1509             if not isinstance(runs, list):
1510                 return
1511             return ''.join([r['text'] for r in runs if isinstance(r.get('text'), compat_str)])
1512
1513         search_meta = (
1514             lambda x: self._html_search_meta(x, webpage, default=None)) \
1515             if webpage else lambda x: None
1516
1517         video_details = player_response.get('videoDetails') or {}
1518         microformat = try_get(
1519             player_response,
1520             lambda x: x['microformat']['playerMicroformatRenderer'],
1521             dict) or {}
1522         video_title = video_details.get('title') \
1523             or get_text(microformat.get('title')) \
1524             or search_meta(['og:title', 'twitter:title', 'title'])
1525         video_description = video_details.get('shortDescription')
1526
1527         if not smuggled_data.get('force_singlefeed', False):
1528             if not self._downloader.params.get('noplaylist'):
1529                 multifeed_metadata_list = try_get(
1530                     player_response,
1531                     lambda x: x['multicamera']['playerLegacyMulticameraRenderer']['metadataList'],
1532                     compat_str)
1533                 if multifeed_metadata_list:
1534                     entries = []
1535                     feed_ids = []
1536                     for feed in multifeed_metadata_list.split(','):
1537                         # Unquote should take place before split on comma (,) since textual
1538                         # fields may contain comma as well (see
1539                         # https://github.com/ytdl-org/youtube-dl/issues/8536)
1540                         feed_data = compat_parse_qs(
1541                             compat_urllib_parse_unquote_plus(feed))
1542
1543                         def feed_entry(name):
1544                             return try_get(
1545                                 feed_data, lambda x: x[name][0], compat_str)
1546
1547                         feed_id = feed_entry('id')
1548                         if not feed_id:
1549                             continue
1550                         feed_title = feed_entry('title')
1551                         title = video_title
1552                         if feed_title:
1553                             title += ' (%s)' % feed_title
1554                         entries.append({
1555                             '_type': 'url_transparent',
1556                             'ie_key': 'Youtube',
1557                             'url': smuggle_url(
1558                                 base_url + 'watch?v=' + feed_data['id'][0],
1559                                 {'force_singlefeed': True}),
1560                             'title': title,
1561                         })
1562                         feed_ids.append(feed_id)
1563                     self.to_screen(
1564                         'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1565                         % (', '.join(feed_ids), video_id))
1566                     return self.playlist_result(
1567                         entries, video_id, video_title, video_description)
1568             else:
1569                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1570
1571         formats = []
1572         itags = []
1573         itag_qualities = {}
1574         player_url = None
1575         q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
1576         streaming_data = player_response.get('streamingData') or {}
1577         streaming_formats = streaming_data.get('formats') or []
1578         streaming_formats.extend(streaming_data.get('adaptiveFormats') or [])
1579         for fmt in streaming_formats:
1580             if fmt.get('targetDurationSec') or fmt.get('drmFamilies'):
1581                 continue
1582
1583             itag = str_or_none(fmt.get('itag'))
1584             quality = fmt.get('quality')
1585             if itag and quality:
1586                 itag_qualities[itag] = quality
1587             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
1588             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
1589             # number of fragment that would subsequently requested with (`&sq=N`)
1590             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
1591                 continue
1592
1593             fmt_url = fmt.get('url')
1594             if not fmt_url:
1595                 sc = compat_parse_qs(fmt.get('signatureCipher'))
1596                 fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
1597                 encrypted_sig = try_get(sc, lambda x: x['s'][0])
1598                 if not (sc and fmt_url and encrypted_sig):
1599                     continue
1600                 if not player_url:
1601                     if not webpage:
1602                         continue
1603                     player_url = self._search_regex(
1604                         r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
1605                         webpage, 'player URL', fatal=False)
1606                 if not player_url:
1607                     continue
1608                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
1609                 sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
1610                 fmt_url += '&' + sp + '=' + signature
1611
1612             if itag:
1613                 itags.append(itag)
1614             tbr = float_or_none(
1615                 fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
1616             dct = {
1617                 'asr': int_or_none(fmt.get('audioSampleRate')),
1618                 'filesize': int_or_none(fmt.get('contentLength')),
1619                 'format_id': itag,
1620                 'format_note': fmt.get('qualityLabel') or quality,
1621                 'fps': int_or_none(fmt.get('fps')),
1622                 'height': int_or_none(fmt.get('height')),
1623                 'quality': q(quality),
1624                 'tbr': tbr,
1625                 'url': fmt_url,
1626                 'width': fmt.get('width'),
1627             }
1628             mimetype = fmt.get('mimeType')
1629             if mimetype:
1630                 mobj = re.match(
1631                     r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', mimetype)
1632                 if mobj:
1633                     dct['ext'] = mimetype2ext(mobj.group(1))
1634                     dct.update(parse_codecs(mobj.group(2)))
1635             no_audio = dct.get('acodec') == 'none'
1636             no_video = dct.get('vcodec') == 'none'
1637             if no_audio:
1638                 dct['vbr'] = tbr
1639             if no_video:
1640                 dct['abr'] = tbr
1641             if no_audio or no_video:
1642                 dct['downloader_options'] = {
1643                     # Youtube throttles chunks >~10M
1644                     'http_chunk_size': 10485760,
1645                 }
1646                 if dct.get('ext'):
1647                     dct['container'] = dct['ext'] + '_dash'
1648             formats.append(dct)
1649
1650         hls_manifest_url = streaming_data.get('hlsManifestUrl')
1651         if hls_manifest_url:
1652             for f in self._extract_m3u8_formats(
1653                     hls_manifest_url, video_id, 'mp4', fatal=False):
1654                 itag = self._search_regex(
1655                     r'/itag/(\d+)', f['url'], 'itag', default=None)
1656                 if itag:
1657                     f['format_id'] = itag
1658                 formats.append(f)
1659
1660         if self._downloader.params.get('youtube_include_dash_manifest', True):
1661             dash_manifest_url = streaming_data.get('dashManifestUrl')
1662             if dash_manifest_url:
1663                 for f in self._extract_mpd_formats(
1664                         dash_manifest_url, video_id, fatal=False):
1665                     itag = f['format_id']
1666                     if itag in itags:
1667                         continue
1668                     if itag in itag_qualities:
1669                         f['quality'] = q(itag_qualities[itag])
1670                     filesize = int_or_none(self._search_regex(
1671                         r'/clen/(\d+)', f.get('fragment_base_url')
1672                         or f['url'], 'file size', default=None))
1673                     if filesize:
1674                         f['filesize'] = filesize
1675                     formats.append(f)
1676
1677         if not formats:
1678             if streaming_data.get('licenseInfos'):
1679                 raise ExtractorError(
1680                     'This video is DRM protected.', expected=True)
1681             pemr = try_get(
1682                 playability_status,
1683                 lambda x: x['errorScreen']['playerErrorMessageRenderer'],
1684                 dict) or {}
1685             reason = get_text(pemr.get('reason')) or playability_status.get('reason')
1686             subreason = pemr.get('subreason')
1687             if subreason:
1688                 subreason = clean_html(get_text(subreason))
1689                 if subreason == 'The uploader has not made this video available in your country.':
1690                     countries = microformat.get('availableCountries')
1691                     if not countries:
1692                         regions_allowed = search_meta('regionsAllowed')
1693                         countries = regions_allowed.split(',') if regions_allowed else None
1694                     self.raise_geo_restricted(
1695                         subreason, countries)
1696                 reason += '\n' + subreason
1697             if reason:
1698                 raise ExtractorError(reason, expected=True)
1699
1700         self._sort_formats(formats)
1701
1702         keywords = video_details.get('keywords') or []
1703         if not keywords and webpage:
1704             keywords = [
1705                 unescapeHTML(m.group('content'))
1706                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
1707         for keyword in keywords:
1708             if keyword.startswith('yt:stretch='):
1709                 w, h = keyword.split('=')[1].split(':')
1710                 w, h = int(w), int(h)
1711                 if w > 0 and h > 0:
1712                     ratio = w / h
1713                     for f in formats:
1714                         if f.get('vcodec') != 'none':
1715                             f['stretched_ratio'] = ratio
1716
1717         thumbnails = []
1718         for container in (video_details, microformat):
1719             for thumbnail in (try_get(
1720                     container,
1721                     lambda x: x['thumbnail']['thumbnails'], list) or []):
1722                 thumbnail_url = thumbnail.get('url')
1723                 if not thumbnail_url:
1724                     continue
1725                 thumbnails.append({
1726                     'height': int_or_none(thumbnail.get('height')),
1727                     'url': thumbnail_url,
1728                     'width': int_or_none(thumbnail.get('width')),
1729                 })
1730             if thumbnails:
1731                 break
1732         else:
1733             thumbnail = search_meta(['og:image', 'twitter:image'])
1734             if thumbnail:
1735                 thumbnails = [{'url': thumbnail}]
1736
1737         category = microformat.get('category') or search_meta('genre')
1738         channel_id = video_details.get('channelId') \
1739             or microformat.get('externalChannelId') \
1740             or search_meta('channelId')
1741         duration = int_or_none(
1742             video_details.get('lengthSeconds')
1743             or microformat.get('lengthSeconds')) \
1744             or parse_duration(search_meta('duration'))
1745         is_live = video_details.get('isLive')
1746         owner_profile_url = microformat.get('ownerProfileUrl')
1747
1748         info = {
1749             'id': video_id,
1750             'title': self._live_title(video_title) if is_live else video_title,
1751             'formats': formats,
1752             'thumbnails': thumbnails,
1753             'description': video_description,
1754             'upload_date': unified_strdate(
1755                 microformat.get('uploadDate')
1756                 or search_meta('uploadDate')),
1757             'uploader': video_details['author'],
1758             'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
1759             'uploader_url': owner_profile_url,
1760             'channel_id': channel_id,
1761             'channel_url': 'https://www.youtube.com/channel/' + channel_id if channel_id else None,
1762             'duration': duration,
1763             'view_count': int_or_none(
1764                 video_details.get('viewCount')
1765                 or microformat.get('viewCount')
1766                 or search_meta('interactionCount')),
1767             'average_rating': float_or_none(video_details.get('averageRating')),
1768             'age_limit': 18 if (
1769                 microformat.get('isFamilySafe') is False
1770                 or search_meta('isFamilyFriendly') == 'false'
1771                 or search_meta('og:restrictions:age') == '18+') else 0,
1772             'webpage_url': webpage_url,
1773             'categories': [category] if category else None,
1774             'tags': keywords,
1775             'is_live': is_live,
1776         }
1777
1778         pctr = try_get(
1779             player_response,
1780             lambda x: x['captions']['playerCaptionsTracklistRenderer'], dict)
1781         if pctr:
1782             def process_language(container, base_url, lang_code, query):
1783                 lang_subs = []
1784                 for fmt in self._SUBTITLE_FORMATS:
1785                     query.update({
1786                         'fmt': fmt,
1787                     })
1788                     lang_subs.append({
1789                         'ext': fmt,
1790                         'url': update_url_query(base_url, query),
1791                     })
1792                 container[lang_code] = lang_subs
1793
1794             subtitles = {}
1795             for caption_track in (pctr.get('captionTracks') or []):
1796                 base_url = caption_track.get('baseUrl')
1797                 if not base_url:
1798                     continue
1799                 if caption_track.get('kind') != 'asr':
1800                     lang_code = caption_track.get('languageCode')
1801                     if not lang_code:
1802                         continue
1803                     process_language(
1804                         subtitles, base_url, lang_code, {})
1805                     continue
1806                 automatic_captions = {}
1807                 for translation_language in (pctr.get('translationLanguages') or []):
1808                     translation_language_code = translation_language.get('languageCode')
1809                     if not translation_language_code:
1810                         continue
1811                     process_language(
1812                         automatic_captions, base_url, translation_language_code,
1813                         {'tlang': translation_language_code})
1814                 info['automatic_captions'] = automatic_captions
1815             info['subtitles'] = subtitles
1816
1817         parsed_url = compat_urllib_parse_urlparse(url)
1818         for component in [parsed_url.fragment, parsed_url.query]:
1819             query = compat_parse_qs(component)
1820             for k, v in query.items():
1821                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
1822                     d_k += '_time'
1823                     if d_k not in info and k in s_ks:
1824                         info[d_k] = parse_duration(query[k][0])
1825
1826         if video_description:
1827             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
1828             if mobj:
1829                 release_year = mobj.group('release_year')
1830                 release_date = mobj.group('release_date')
1831                 if release_date:
1832                     release_date = release_date.replace('-', '')
1833                     if not release_year:
1834                         release_year = release_date[:4]
1835                 info.update({
1836                     'album': mobj.group('album'.strip()),
1837                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
1838                     'track': mobj.group('track').strip(),
1839                     'release_date': release_date,
1840                     'release_year': int_or_none(release_year),
1841                 })
1842
1843         initial_data = None
1844         if webpage:
1845             initial_data = self._extract_yt_initial_variable(
1846                 webpage, self._YT_INITIAL_DATA_RE, video_id,
1847                 'yt initial data')
1848         if not initial_data:
1849             initial_data = self._call_api(
1850                 'next', {'videoId': video_id}, video_id, fatal=False)
1851
1852         if initial_data:
1853             chapters = self._extract_chapters_from_json(
1854                 initial_data, video_id, duration)
1855             if not chapters:
1856                 for engagment_pannel in (initial_data.get('engagementPanels') or []):
1857                     contents = try_get(
1858                         engagment_pannel, lambda x: x['engagementPanelSectionListRenderer']['content']['macroMarkersListRenderer']['contents'],
1859                         list)
1860                     if not contents:
1861                         continue
1862
1863                     def chapter_time(mmlir):
1864                         return parse_duration(
1865                             get_text(mmlir.get('timeDescription')))
1866
1867                     chapters = []
1868                     for next_num, content in enumerate(contents, start=1):
1869                         mmlir = content.get('macroMarkersListItemRenderer') or {}
1870                         start_time = chapter_time(mmlir)
1871                         end_time = chapter_time(try_get(
1872                             contents, lambda x: x[next_num]['macroMarkersListItemRenderer'])) \
1873                             if next_num < len(contents) else duration
1874                         if start_time is None or end_time is None:
1875                             continue
1876                         chapters.append({
1877                             'start_time': start_time,
1878                             'end_time': end_time,
1879                             'title': get_text(mmlir.get('title')),
1880                         })
1881                     if chapters:
1882                         break
1883             if chapters:
1884                 info['chapters'] = chapters
1885
1886             contents = try_get(
1887                 initial_data,
1888                 lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'],
1889                 list) or []
1890             for content in contents:
1891                 vpir = content.get('videoPrimaryInfoRenderer')
1892                 if vpir:
1893                     stl = vpir.get('superTitleLink')
1894                     if stl:
1895                         stl = get_text(stl)
1896                         if try_get(
1897                                 vpir,
1898                                 lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
1899                             info['location'] = stl
1900                         else:
1901                             mobj = re.search(r'(.+?)\s*S(\d+)\s*•\s*E(\d+)', stl)
1902                             if mobj:
1903                                 info.update({
1904                                     'series': mobj.group(1),
1905                                     'season_number': int(mobj.group(2)),
1906                                     'episode_number': int(mobj.group(3)),
1907                                 })
1908                     for tlb in (try_get(
1909                             vpir,
1910                             lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
1911                             list) or []):
1912                         tbr = tlb.get('toggleButtonRenderer') or {}
1913                         for getter, regex in [(
1914                                 lambda x: x['defaultText']['accessibility']['accessibilityData'],
1915                                 r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
1916                                     lambda x: x['accessibility'],
1917                                     lambda x: x['accessibilityData']['accessibilityData'],
1918                                 ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
1919                             label = (try_get(tbr, getter, dict) or {}).get('label')
1920                             if label:
1921                                 mobj = re.match(regex, label)
1922                                 if mobj:
1923                                     info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
1924                                     break
1925                     sbr_tooltip = try_get(
1926                         vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
1927                     if sbr_tooltip:
1928                         like_count, dislike_count = sbr_tooltip.split(' / ')
1929                         info.update({
1930                             'like_count': str_to_int(like_count),
1931                             'dislike_count': str_to_int(dislike_count),
1932                         })
1933                 vsir = content.get('videoSecondaryInfoRenderer')
1934                 if vsir:
1935                     info['channel'] = get_text(try_get(
1936                         vsir,
1937                         lambda x: x['owner']['videoOwnerRenderer']['title'],
1938                         dict))
1939                     rows = try_get(
1940                         vsir,
1941                         lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
1942                         list) or []
1943                     multiple_songs = False
1944                     for row in rows:
1945                         if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
1946                             multiple_songs = True
1947                             break
1948                     for row in rows:
1949                         mrr = row.get('metadataRowRenderer') or {}
1950                         mrr_title = mrr.get('title')
1951                         if not mrr_title:
1952                             continue
1953                         mrr_title = get_text(mrr['title'])
1954                         mrr_contents_text = get_text(mrr['contents'][0])
1955                         if mrr_title == 'License':
1956                             info['license'] = mrr_contents_text
1957                         elif not multiple_songs:
1958                             if mrr_title == 'Album':
1959                                 info['album'] = mrr_contents_text
1960                             elif mrr_title == 'Artist':
1961                                 info['artist'] = mrr_contents_text
1962                             elif mrr_title == 'Song':
1963                                 info['track'] = mrr_contents_text
1964
1965         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
1966             v = info.get(s_k)
1967             if v:
1968                 info[d_k] = v
1969
1970         self.mark_watched(video_id, player_response)
1971
1972         return info
1973
1974
1975 class YoutubeTabIE(YoutubeBaseInfoExtractor):
1976     IE_DESC = 'YouTube.com tab'
1977     _VALID_URL = r'''(?x)
1978                     https?://
1979                         (?:\w+\.)?
1980                         (?:
1981                             youtube(?:kids)?\.com|
1982                             invidio\.us
1983                         )/
1984                         (?:
1985                             (?:channel|c|user|feed|hashtag)/|
1986                             (?:playlist|watch)\?.*?\blist=|
1987                             (?!(?:watch|embed|v|e)\b)
1988                         )
1989                         (?P<id>[^/?\#&]+)
1990                     '''
1991     IE_NAME = 'youtube:tab'
1992
1993     _TESTS = [{
1994         # playlists, multipage
1995         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
1996         'playlist_mincount': 94,
1997         'info_dict': {
1998             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
1999             'title': 'Игорь Клейнер - Playlists',
2000             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
2001         },
2002     }, {
2003         # playlists, multipage, different order
2004         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
2005         'playlist_mincount': 94,
2006         'info_dict': {
2007             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
2008             'title': 'Игорь Клейнер - Playlists',
2009             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
2010         },
2011     }, {
2012         # playlists, singlepage
2013         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
2014         'playlist_mincount': 4,
2015         'info_dict': {
2016             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
2017             'title': 'ThirstForScience - Playlists',
2018             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
2019         }
2020     }, {
2021         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
2022         'only_matching': True,
2023     }, {
2024         # basic, single video playlist
2025         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2026         'info_dict': {
2027             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
2028             'uploader': 'Sergey M.',
2029             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2030             'title': 'youtube-dl public playlist',
2031         },
2032         'playlist_count': 1,
2033     }, {
2034         # empty playlist
2035         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
2036         'info_dict': {
2037             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
2038             'uploader': 'Sergey M.',
2039             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
2040             'title': 'youtube-dl empty playlist',
2041         },
2042         'playlist_count': 0,
2043     }, {
2044         # Home tab
2045         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
2046         'info_dict': {
2047             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2048             'title': 'lex will - Home',
2049             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2050         },
2051         'playlist_mincount': 2,
2052     }, {
2053         # Videos tab
2054         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
2055         'info_dict': {
2056             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2057             'title': 'lex will - Videos',
2058             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2059         },
2060         'playlist_mincount': 975,
2061     }, {
2062         # Videos tab, sorted by popular
2063         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
2064         'info_dict': {
2065             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2066             'title': 'lex will - Videos',
2067             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2068         },
2069         'playlist_mincount': 199,
2070     }, {
2071         # Playlists tab
2072         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
2073         'info_dict': {
2074             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2075             'title': 'lex will - Playlists',
2076             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2077         },
2078         'playlist_mincount': 17,
2079     }, {
2080         # Community tab
2081         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
2082         'info_dict': {
2083             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2084             'title': 'lex will - Community',
2085             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2086         },
2087         'playlist_mincount': 18,
2088     }, {
2089         # Channels tab
2090         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
2091         'info_dict': {
2092             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2093             'title': 'lex will - Channels',
2094             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
2095         },
2096         'playlist_mincount': 138,
2097     }, {
2098         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
2099         'only_matching': True,
2100     }, {
2101         'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
2102         'only_matching': True,
2103     }, {
2104         'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
2105         'only_matching': True,
2106     }, {
2107         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
2108         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2109         'info_dict': {
2110             'title': '29C3: Not my department',
2111             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2112             'uploader': 'Christiaan008',
2113             'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
2114         },
2115         'playlist_count': 96,
2116     }, {
2117         'note': 'Large playlist',
2118         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
2119         'info_dict': {
2120             'title': 'Uploads from Cauchemar',
2121             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
2122             'uploader': 'Cauchemar',
2123             'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
2124         },
2125         'playlist_mincount': 1123,
2126     }, {
2127         # even larger playlist, 8832 videos
2128         'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
2129         'only_matching': True,
2130     }, {
2131         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
2132         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
2133         'info_dict': {
2134             'title': 'Uploads from Interstellar Movie',
2135             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
2136             'uploader': 'Interstellar Movie',
2137             'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
2138         },
2139         'playlist_mincount': 21,
2140     }, {
2141         # https://github.com/ytdl-org/youtube-dl/issues/21844
2142         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2143         'info_dict': {
2144             'title': 'Data Analysis with Dr Mike Pound',
2145             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2146             'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
2147             'uploader': 'Computerphile',
2148         },
2149         'playlist_mincount': 11,
2150     }, {
2151         'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2152         'only_matching': True,
2153     }, {
2154         # Playlist URL that does not actually serve a playlist
2155         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
2156         'info_dict': {
2157             'id': 'FqZTN594JQw',
2158             'ext': 'webm',
2159             'title': "Smiley's People 01 detective, Adventure Series, Action",
2160             'uploader': 'STREEM',
2161             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
2162             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
2163             'upload_date': '20150526',
2164             'license': 'Standard YouTube License',
2165             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
2166             'categories': ['People & Blogs'],
2167             'tags': list,
2168             'view_count': int,
2169             'like_count': int,
2170             'dislike_count': int,
2171         },
2172         'params': {
2173             'skip_download': True,
2174         },
2175         'skip': 'This video is not available.',
2176         'add_ie': [YoutubeIE.ie_key()],
2177     }, {
2178         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
2179         'only_matching': True,
2180     }, {
2181         'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
2182         'only_matching': True,
2183     }, {
2184         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
2185         'info_dict': {
2186             'id': '9Auq9mYxFEE',
2187             'ext': 'mp4',
2188             'title': 'Watch Sky News live',
2189             'uploader': 'Sky News',
2190             'uploader_id': 'skynews',
2191             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
2192             'upload_date': '20191102',
2193             'description': 'md5:78de4e1c2359d0ea3ed829678e38b662',
2194             'categories': ['News & Politics'],
2195             'tags': list,
2196             'like_count': int,
2197             'dislike_count': int,
2198         },
2199         'params': {
2200             'skip_download': True,
2201         },
2202     }, {
2203         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
2204         'info_dict': {
2205             'id': 'a48o2S1cPoo',
2206             'ext': 'mp4',
2207             'title': 'The Young Turks - Live Main Show',
2208             'uploader': 'The Young Turks',
2209             'uploader_id': 'TheYoungTurks',
2210             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
2211             'upload_date': '20150715',
2212             'license': 'Standard YouTube License',
2213             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
2214             'categories': ['News & Politics'],
2215             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
2216             'like_count': int,
2217             'dislike_count': int,
2218         },
2219         'params': {
2220             'skip_download': True,
2221         },
2222         'only_matching': True,
2223     }, {
2224         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
2225         'only_matching': True,
2226     }, {
2227         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
2228         'only_matching': True,
2229     }, {
2230         'url': 'https://www.youtube.com/feed/trending',
2231         'only_matching': True,
2232     }, {
2233         # needs auth
2234         'url': 'https://www.youtube.com/feed/library',
2235         'only_matching': True,
2236     }, {
2237         # needs auth
2238         'url': 'https://www.youtube.com/feed/history',
2239         'only_matching': True,
2240     }, {
2241         # needs auth
2242         'url': 'https://www.youtube.com/feed/subscriptions',
2243         'only_matching': True,
2244     }, {
2245         # needs auth
2246         'url': 'https://www.youtube.com/feed/watch_later',
2247         'only_matching': True,
2248     }, {
2249         # no longer available?
2250         'url': 'https://www.youtube.com/feed/recommended',
2251         'only_matching': True,
2252     }, {
2253         # inline playlist with not always working continuations
2254         'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
2255         'only_matching': True,
2256     }, {
2257         'url': 'https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8',
2258         'only_matching': True,
2259     }, {
2260         'url': 'https://www.youtube.com/course',
2261         'only_matching': True,
2262     }, {
2263         'url': 'https://www.youtube.com/zsecurity',
2264         'only_matching': True,
2265     }, {
2266         'url': 'http://www.youtube.com/NASAgovVideo/videos',
2267         'only_matching': True,
2268     }, {
2269         'url': 'https://www.youtube.com/TheYoungTurks/live',
2270         'only_matching': True,
2271     }, {
2272         'url': 'https://www.youtube.com/hashtag/cctv9',
2273         'info_dict': {
2274             'id': 'cctv9',
2275             'title': '#cctv9',
2276         },
2277         'playlist_mincount': 350,
2278     }]
2279
2280     @classmethod
2281     def suitable(cls, url):
2282         return False if YoutubeIE.suitable(url) else super(
2283             YoutubeTabIE, cls).suitable(url)
2284
2285     def _extract_channel_id(self, webpage):
2286         channel_id = self._html_search_meta(
2287             'channelId', webpage, 'channel id', default=None)
2288         if channel_id:
2289             return channel_id
2290         channel_url = self._html_search_meta(
2291             ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
2292              'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
2293              'twitter:app:url:googleplay'), webpage, 'channel url')
2294         return self._search_regex(
2295             r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
2296             channel_url, 'channel id')
2297
2298     @staticmethod
2299     def _extract_grid_item_renderer(item):
2300         for item_kind in ('Playlist', 'Video', 'Channel'):
2301             renderer = item.get('grid%sRenderer' % item_kind)
2302             if renderer:
2303                 return renderer
2304
2305     def _grid_entries(self, grid_renderer):
2306         for item in grid_renderer['items']:
2307             if not isinstance(item, dict):
2308                 continue
2309             renderer = self._extract_grid_item_renderer(item)
2310             if not isinstance(renderer, dict):
2311                 continue
2312             title = try_get(
2313                 renderer, lambda x: x['title']['runs'][0]['text'], compat_str)
2314             # playlist
2315             playlist_id = renderer.get('playlistId')
2316             if playlist_id:
2317                 yield self.url_result(
2318                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
2319                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
2320                     video_title=title)
2321             # video
2322             video_id = renderer.get('videoId')
2323             if video_id:
2324                 yield self._extract_video(renderer)
2325             # channel
2326             channel_id = renderer.get('channelId')
2327             if channel_id:
2328                 title = try_get(
2329                     renderer, lambda x: x['title']['simpleText'], compat_str)
2330                 yield self.url_result(
2331                     'https://www.youtube.com/channel/%s' % channel_id,
2332                     ie=YoutubeTabIE.ie_key(), video_title=title)
2333
2334     def _shelf_entries_from_content(self, shelf_renderer):
2335         content = shelf_renderer.get('content')
2336         if not isinstance(content, dict):
2337             return
2338         renderer = content.get('gridRenderer')
2339         if renderer:
2340             # TODO: add support for nested playlists so each shelf is processed
2341             # as separate playlist
2342             # TODO: this includes only first N items
2343             for entry in self._grid_entries(renderer):
2344                 yield entry
2345         renderer = content.get('horizontalListRenderer')
2346         if renderer:
2347             # TODO
2348             pass
2349
2350     def _shelf_entries(self, shelf_renderer, skip_channels=False):
2351         ep = try_get(
2352             shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
2353             compat_str)
2354         shelf_url = urljoin('https://www.youtube.com', ep)
2355         if shelf_url:
2356             # Skipping links to another channels, note that checking for
2357             # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
2358             # will not work
2359             if skip_channels and '/channels?' in shelf_url:
2360                 return
2361             title = try_get(
2362                 shelf_renderer, lambda x: x['title']['runs'][0]['text'], compat_str)
2363             yield self.url_result(shelf_url, video_title=title)
2364         # Shelf may not contain shelf URL, fallback to extraction from content
2365         for entry in self._shelf_entries_from_content(shelf_renderer):
2366             yield entry
2367
2368     def _playlist_entries(self, video_list_renderer):
2369         for content in video_list_renderer['contents']:
2370             if not isinstance(content, dict):
2371                 continue
2372             renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
2373             if not isinstance(renderer, dict):
2374                 continue
2375             video_id = renderer.get('videoId')
2376             if not video_id:
2377                 continue
2378             yield self._extract_video(renderer)
2379
2380     def _video_entry(self, video_renderer):
2381         video_id = video_renderer.get('videoId')
2382         if video_id:
2383             return self._extract_video(video_renderer)
2384
2385     def _post_thread_entries(self, post_thread_renderer):
2386         post_renderer = try_get(
2387             post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
2388         if not post_renderer:
2389             return
2390         # video attachment
2391         video_renderer = try_get(
2392             post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict)
2393         video_id = None
2394         if video_renderer:
2395             entry = self._video_entry(video_renderer)
2396             if entry:
2397                 yield entry
2398         # inline video links
2399         runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
2400         for run in runs:
2401             if not isinstance(run, dict):
2402                 continue
2403             ep_url = try_get(
2404                 run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], compat_str)
2405             if not ep_url:
2406                 continue
2407             if not YoutubeIE.suitable(ep_url):
2408                 continue
2409             ep_video_id = YoutubeIE._match_id(ep_url)
2410             if video_id == ep_video_id:
2411                 continue
2412             yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=video_id)
2413
2414     def _post_thread_continuation_entries(self, post_thread_continuation):
2415         contents = post_thread_continuation.get('contents')
2416         if not isinstance(contents, list):
2417             return
2418         for content in contents:
2419             renderer = content.get('backstagePostThreadRenderer')
2420             if not isinstance(renderer, dict):
2421                 continue
2422             for entry in self._post_thread_entries(renderer):
2423                 yield entry
2424
2425     def _rich_grid_entries(self, contents):
2426         for content in contents:
2427             video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
2428             if video_renderer:
2429                 entry = self._video_entry(video_renderer)
2430                 if entry:
2431                     yield entry
2432
2433     @staticmethod
2434     def _build_continuation_query(continuation, ctp=None):
2435         query = {
2436             'ctoken': continuation,
2437             'continuation': continuation,
2438         }
2439         if ctp:
2440             query['itct'] = ctp
2441         return query
2442
2443     @staticmethod
2444     def _extract_next_continuation_data(renderer):
2445         next_continuation = try_get(
2446             renderer, lambda x: x['continuations'][0]['nextContinuationData'], dict)
2447         if not next_continuation:
2448             return
2449         continuation = next_continuation.get('continuation')
2450         if not continuation:
2451             return
2452         ctp = next_continuation.get('clickTrackingParams')
2453         return YoutubeTabIE._build_continuation_query(continuation, ctp)
2454
2455     @classmethod
2456     def _extract_continuation(cls, renderer):
2457         next_continuation = cls._extract_next_continuation_data(renderer)
2458         if next_continuation:
2459             return next_continuation
2460         contents = []
2461         for key in ('contents', 'items'):
2462             contents.extend(try_get(renderer, lambda x: x[key], list) or [])
2463         for content in contents:
2464             if not isinstance(content, dict):
2465                 continue
2466             continuation_ep = try_get(
2467                 content, lambda x: x['continuationItemRenderer']['continuationEndpoint'],
2468                 dict)
2469             if not continuation_ep:
2470                 continue
2471             continuation = try_get(
2472                 continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
2473             if not continuation:
2474                 continue
2475             ctp = continuation_ep.get('clickTrackingParams')
2476             return YoutubeTabIE._build_continuation_query(continuation, ctp)
2477
2478     def _entries(self, tab, identity_token):
2479         tab_content = try_get(tab, lambda x: x['content'], dict)
2480         if not tab_content:
2481             return
2482         slr_renderer = try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
2483         if slr_renderer:
2484             is_channels_tab = tab.get('title') == 'Channels'
2485             continuation = None
2486             slr_contents = try_get(slr_renderer, lambda x: x['contents'], list) or []
2487             for slr_content in slr_contents:
2488                 if not isinstance(slr_content, dict):
2489                     continue
2490                 is_renderer = try_get(slr_content, lambda x: x['itemSectionRenderer'], dict)
2491                 if not is_renderer:
2492                     continue
2493                 isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
2494                 for isr_content in isr_contents:
2495                     if not isinstance(isr_content, dict):
2496                         continue
2497                     renderer = isr_content.get('playlistVideoListRenderer')
2498                     if renderer:
2499                         for entry in self._playlist_entries(renderer):
2500                             yield entry
2501                         continuation = self._extract_continuation(renderer)
2502                         continue
2503                     renderer = isr_content.get('gridRenderer')
2504                     if renderer:
2505                         for entry in self._grid_entries(renderer):
2506                             yield entry
2507                         continuation = self._extract_continuation(renderer)
2508                         continue
2509                     renderer = isr_content.get('shelfRenderer')
2510                     if renderer:
2511                         for entry in self._shelf_entries(renderer, not is_channels_tab):
2512                             yield entry
2513                         continue
2514                     renderer = isr_content.get('backstagePostThreadRenderer')
2515                     if renderer:
2516                         for entry in self._post_thread_entries(renderer):
2517                             yield entry
2518                         continuation = self._extract_continuation(renderer)
2519                         continue
2520                     renderer = isr_content.get('videoRenderer')
2521                     if renderer:
2522                         entry = self._video_entry(renderer)
2523                         if entry:
2524                             yield entry
2525
2526                 if not continuation:
2527                     continuation = self._extract_continuation(is_renderer)
2528             if not continuation:
2529                 continuation = self._extract_continuation(slr_renderer)
2530         else:
2531             rich_grid_renderer = tab_content.get('richGridRenderer')
2532             if not rich_grid_renderer:
2533                 return
2534             for entry in self._rich_grid_entries(rich_grid_renderer.get('contents') or []):
2535                 yield entry
2536             continuation = self._extract_continuation(rich_grid_renderer)
2537
2538         headers = {
2539             'x-youtube-client-name': '1',
2540             'x-youtube-client-version': '2.20201112.04.01',
2541             'content-type': 'application/json',
2542         }
2543         if identity_token:
2544             headers['x-youtube-identity-token'] = identity_token
2545
2546         data = {
2547             'context': {
2548                 'client': {
2549                     'clientName': 'WEB',
2550                     'clientVersion': '2.20201021.03.00',
2551                 }
2552             },
2553         }
2554
2555         for page_num in itertools.count(1):
2556             if not continuation:
2557                 break
2558             data['continuation'] = continuation['continuation']
2559             data['clickTracking'] = {
2560                 'clickTrackingParams': continuation['itct']
2561             }
2562             count = 0
2563             retries = 3
2564             while count <= retries:
2565                 try:
2566                     # Downloading page may result in intermittent 5xx HTTP error
2567                     # that is usually worked around with a retry
2568                     response = self._download_json(
2569                         'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
2570                         None, 'Downloading page %d%s' % (page_num, ' (retry #%d)' % count if count else ''),
2571                         headers=headers, data=json.dumps(data).encode('utf8'))
2572                     break
2573                 except ExtractorError as e:
2574                     if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503):
2575                         count += 1
2576                         if count <= retries:
2577                             continue
2578                     raise
2579             if not response:
2580                 break
2581
2582             continuation_contents = try_get(
2583                 response, lambda x: x['continuationContents'], dict)
2584             if continuation_contents:
2585                 continuation_renderer = continuation_contents.get('playlistVideoListContinuation')
2586                 if continuation_renderer:
2587                     for entry in self._playlist_entries(continuation_renderer):
2588                         yield entry
2589                     continuation = self._extract_continuation(continuation_renderer)
2590                     continue
2591                 continuation_renderer = continuation_contents.get('gridContinuation')
2592                 if continuation_renderer:
2593                     for entry in self._grid_entries(continuation_renderer):
2594                         yield entry
2595                     continuation = self._extract_continuation(continuation_renderer)
2596                     continue
2597                 continuation_renderer = continuation_contents.get('itemSectionContinuation')
2598                 if continuation_renderer:
2599                     for entry in self._post_thread_continuation_entries(continuation_renderer):
2600                         yield entry
2601                     continuation = self._extract_continuation(continuation_renderer)
2602                     continue
2603
2604             on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
2605             continuation_items = try_get(
2606                 on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
2607             if continuation_items:
2608                 continuation_item = continuation_items[0]
2609                 if not isinstance(continuation_item, dict):
2610                     continue
2611                 renderer = self._extract_grid_item_renderer(continuation_item)
2612                 if renderer:
2613                     grid_renderer = {'items': continuation_items}
2614                     for entry in self._grid_entries(grid_renderer):
2615                         yield entry
2616                     continuation = self._extract_continuation(grid_renderer)
2617                     continue
2618                 renderer = continuation_item.get('playlistVideoRenderer') or continuation_item.get('itemSectionRenderer')
2619                 if renderer:
2620                     video_list_renderer = {'contents': continuation_items}
2621                     for entry in self._playlist_entries(video_list_renderer):
2622                         yield entry
2623                     continuation = self._extract_continuation(video_list_renderer)
2624                     continue
2625                 renderer = continuation_item.get('backstagePostThreadRenderer')
2626                 if renderer:
2627                     continuation_renderer = {'contents': continuation_items}
2628                     for entry in self._post_thread_continuation_entries(continuation_renderer):
2629                         yield entry
2630                     continuation = self._extract_continuation(continuation_renderer)
2631                     continue
2632                 renderer = continuation_item.get('richItemRenderer')
2633                 if renderer:
2634                     for entry in self._rich_grid_entries(continuation_items):
2635                         yield entry
2636                     continuation = self._extract_continuation({'contents': continuation_items})
2637                     continue
2638
2639             break
2640
2641     @staticmethod
2642     def _extract_selected_tab(tabs):
2643         for tab in tabs:
2644             if try_get(tab, lambda x: x['tabRenderer']['selected'], bool):
2645                 return tab['tabRenderer']
2646         else:
2647             raise ExtractorError('Unable to find selected tab')
2648
2649     @staticmethod
2650     def _extract_uploader(data):
2651         uploader = {}
2652         sidebar_renderer = try_get(
2653             data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list)
2654         if sidebar_renderer:
2655             for item in sidebar_renderer:
2656                 if not isinstance(item, dict):
2657                     continue
2658                 renderer = item.get('playlistSidebarSecondaryInfoRenderer')
2659                 if not isinstance(renderer, dict):
2660                     continue
2661                 owner = try_get(
2662                     renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
2663                 if owner:
2664                     uploader['uploader'] = owner.get('text')
2665                     uploader['uploader_id'] = try_get(
2666                         owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
2667                     uploader['uploader_url'] = urljoin(
2668                         'https://www.youtube.com/',
2669                         try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
2670         return uploader
2671
2672     @staticmethod
2673     def _extract_alert(data):
2674         alerts = []
2675         for alert in try_get(data, lambda x: x['alerts'], list) or []:
2676             if not isinstance(alert, dict):
2677                 continue
2678             alert_text = try_get(
2679                 alert, lambda x: x['alertRenderer']['text'], dict)
2680             if not alert_text:
2681                 continue
2682             text = try_get(
2683                 alert_text,
2684                 (lambda x: x['simpleText'], lambda x: x['runs'][0]['text']),
2685                 compat_str)
2686             if text:
2687                 alerts.append(text)
2688         return '\n'.join(alerts)
2689
2690     def _extract_from_tabs(self, item_id, webpage, data, tabs, identity_token):
2691         selected_tab = self._extract_selected_tab(tabs)
2692         renderer = try_get(
2693             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
2694         playlist_id = item_id
2695         title = description = None
2696         if renderer:
2697             channel_title = renderer.get('title') or item_id
2698             tab_title = selected_tab.get('title')
2699             title = channel_title or item_id
2700             if tab_title:
2701                 title += ' - %s' % tab_title
2702             description = renderer.get('description')
2703             playlist_id = renderer.get('externalId')
2704         else:
2705             renderer = try_get(
2706                 data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
2707             if renderer:
2708                 title = renderer.get('title')
2709             else:
2710                 renderer = try_get(
2711                     data, lambda x: x['header']['hashtagHeaderRenderer'], dict)
2712                 if renderer:
2713                     title = try_get(renderer, lambda x: x['hashtag']['simpleText'])
2714         playlist = self.playlist_result(
2715             self._entries(selected_tab, identity_token),
2716             playlist_id=playlist_id, playlist_title=title,
2717             playlist_description=description)
2718         playlist.update(self._extract_uploader(data))
2719         return playlist
2720
2721     def _extract_from_playlist(self, item_id, url, data, playlist):
2722         title = playlist.get('title') or try_get(
2723             data, lambda x: x['titleText']['simpleText'], compat_str)
2724         playlist_id = playlist.get('playlistId') or item_id
2725         # Inline playlist rendition continuation does not always work
2726         # at Youtube side, so delegating regular tab-based playlist URL
2727         # processing whenever possible.
2728         playlist_url = urljoin(url, try_get(
2729             playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
2730             compat_str))
2731         if playlist_url and playlist_url != url:
2732             return self.url_result(
2733                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
2734                 video_title=title)
2735         return self.playlist_result(
2736             self._playlist_entries(playlist), playlist_id=playlist_id,
2737             playlist_title=title)
2738
2739     def _extract_identity_token(self, webpage, item_id):
2740         ytcfg = self._extract_ytcfg(item_id, webpage)
2741         if ytcfg:
2742             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
2743             if token:
2744                 return token
2745         return self._search_regex(
2746             r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
2747             'identity token', default=None)
2748
2749     def _real_extract(self, url):
2750         item_id = self._match_id(url)
2751         url = compat_urlparse.urlunparse(
2752             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
2753         # Handle both video/playlist URLs
2754         qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
2755         video_id = qs.get('v', [None])[0]
2756         playlist_id = qs.get('list', [None])[0]
2757         if video_id and playlist_id:
2758             if self._downloader.params.get('noplaylist'):
2759                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
2760                 return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
2761             self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
2762         webpage = self._download_webpage(url, item_id)
2763         identity_token = self._extract_identity_token(webpage, item_id)
2764         data = self._extract_yt_initial_data(item_id, webpage)
2765         tabs = try_get(
2766             data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
2767         if tabs:
2768             return self._extract_from_tabs(item_id, webpage, data, tabs, identity_token)
2769         playlist = try_get(
2770             data, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
2771         if playlist:
2772             return self._extract_from_playlist(item_id, url, data, playlist)
2773         # Fallback to video extraction if no playlist alike page is recognized.
2774         # First check for the current video then try the v attribute of URL query.
2775         video_id = try_get(
2776             data, lambda x: x['currentVideoEndpoint']['watchEndpoint']['videoId'],
2777             compat_str) or video_id
2778         if video_id:
2779             return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
2780         # Capture and output alerts
2781         alert = self._extract_alert(data)
2782         if alert:
2783             raise ExtractorError(alert, expected=True)
2784         # Failed to recognize
2785         raise ExtractorError('Unable to recognize tab page')
2786
2787
2788 class YoutubePlaylistIE(InfoExtractor):
2789     IE_DESC = 'YouTube.com playlists'
2790     _VALID_URL = r'''(?x)(?:
2791                         (?:https?://)?
2792                         (?:\w+\.)?
2793                         (?:
2794                             (?:
2795                                 youtube(?:kids)?\.com|
2796                                 invidio\.us
2797                             )
2798                             /.*?\?.*?\blist=
2799                         )?
2800                         (?P<id>%(playlist_id)s)
2801                      )''' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
2802     IE_NAME = 'youtube:playlist'
2803     _TESTS = [{
2804         'note': 'issue #673',
2805         'url': 'PLBB231211A4F62143',
2806         'info_dict': {
2807             'title': '[OLD]Team Fortress 2 (Class-based LP)',
2808             'id': 'PLBB231211A4F62143',
2809             'uploader': 'Wickydoo',
2810             'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
2811         },
2812         'playlist_mincount': 29,
2813     }, {
2814         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2815         'info_dict': {
2816             'title': 'YDL_safe_search',
2817             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2818         },
2819         'playlist_count': 2,
2820         'skip': 'This playlist is private',
2821     }, {
2822         'note': 'embedded',
2823         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2824         'playlist_count': 4,
2825         'info_dict': {
2826             'title': 'JODA15',
2827             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2828             'uploader': 'milan',
2829             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
2830         }
2831     }, {
2832         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2833         'playlist_mincount': 982,
2834         'info_dict': {
2835             'title': '2018 Chinese New Singles (11/6 updated)',
2836             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2837             'uploader': 'LBK',
2838             'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
2839         }
2840     }, {
2841         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
2842         'only_matching': True,
2843     }, {
2844         # music album playlist
2845         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
2846         'only_matching': True,
2847     }]
2848
2849     @classmethod
2850     def suitable(cls, url):
2851         return False if YoutubeTabIE.suitable(url) else super(
2852             YoutubePlaylistIE, cls).suitable(url)
2853
2854     def _real_extract(self, url):
2855         playlist_id = self._match_id(url)
2856         qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
2857         if not qs:
2858             qs = {'list': playlist_id}
2859         return self.url_result(
2860             update_url_query('https://www.youtube.com/playlist', qs),
2861             ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
2862
2863
2864 class YoutubeYtBeIE(InfoExtractor):
2865     _VALID_URL = r'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{11})/*?.*?\blist=(?P<playlist_id>%(playlist_id)s)' % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
2866     _TESTS = [{
2867         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
2868         'info_dict': {
2869             'id': 'yeWKywCrFtk',
2870             'ext': 'mp4',
2871             'title': 'Small Scale Baler and Braiding Rugs',
2872             'uploader': 'Backus-Page House Museum',
2873             'uploader_id': 'backuspagemuseum',
2874             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
2875             'upload_date': '20161008',
2876             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
2877             'categories': ['Nonprofits & Activism'],
2878             'tags': list,
2879             'like_count': int,
2880             'dislike_count': int,
2881         },
2882         'params': {
2883             'noplaylist': True,
2884             'skip_download': True,
2885         },
2886     }, {
2887         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
2888         'only_matching': True,
2889     }]
2890
2891     def _real_extract(self, url):
2892         mobj = re.match(self._VALID_URL, url)
2893         video_id = mobj.group('id')
2894         playlist_id = mobj.group('playlist_id')
2895         return self.url_result(
2896             update_url_query('https://www.youtube.com/watch', {
2897                 'v': video_id,
2898                 'list': playlist_id,
2899                 'feature': 'youtu.be',
2900             }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
2901
2902
2903 class YoutubeYtUserIE(InfoExtractor):
2904     _VALID_URL = r'ytuser:(?P<id>.+)'
2905     _TESTS = [{
2906         'url': 'ytuser:phihag',
2907         'only_matching': True,
2908     }]
2909
2910     def _real_extract(self, url):
2911         user_id = self._match_id(url)
2912         return self.url_result(
2913             'https://www.youtube.com/user/%s' % user_id,
2914             ie=YoutubeTabIE.ie_key(), video_id=user_id)
2915
2916
2917 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
2918     IE_NAME = 'youtube:favorites'
2919     IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
2920     _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
2921     _LOGIN_REQUIRED = True
2922     _TESTS = [{
2923         'url': ':ytfav',
2924         'only_matching': True,
2925     }, {
2926         'url': ':ytfavorites',
2927         'only_matching': True,
2928     }]
2929
2930     def _real_extract(self, url):
2931         return self.url_result(
2932             'https://www.youtube.com/playlist?list=LL',
2933             ie=YoutubeTabIE.ie_key())
2934
2935
2936 class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
2937     IE_DESC = 'YouTube.com searches'
2938     # there doesn't appear to be a real limit, for example if you search for
2939     # 'python' you get more than 8.000.000 results
2940     _MAX_RESULTS = float('inf')
2941     IE_NAME = 'youtube:search'
2942     _SEARCH_KEY = 'ytsearch'
2943     _SEARCH_PARAMS = None
2944     _TESTS = []
2945
2946     def _entries(self, query, n):
2947         data = {
2948             'context': {
2949                 'client': {
2950                     'clientName': 'WEB',
2951                     'clientVersion': '2.20201021.03.00',
2952                 }
2953             },
2954             'query': query,
2955         }
2956         if self._SEARCH_PARAMS:
2957             data['params'] = self._SEARCH_PARAMS
2958         total = 0
2959         for page_num in itertools.count(1):
2960             search = self._download_json(
2961                 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
2962                 video_id='query "%s"' % query,
2963                 note='Downloading page %s' % page_num,
2964                 errnote='Unable to download API page', fatal=False,
2965                 data=json.dumps(data).encode('utf8'),
2966                 headers={'content-type': 'application/json'})
2967             if not search:
2968                 break
2969             slr_contents = try_get(
2970                 search,
2971                 (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
2972                  lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
2973                 list)
2974             if not slr_contents:
2975                 break
2976             for slr_content in slr_contents:
2977                 isr_contents = try_get(
2978                     slr_content,
2979                     lambda x: x['itemSectionRenderer']['contents'],
2980                     list)
2981                 if not isr_contents:
2982                     continue
2983                 for content in isr_contents:
2984                     if not isinstance(content, dict):
2985                         continue
2986                     video = content.get('videoRenderer')
2987                     if not isinstance(video, dict):
2988                         continue
2989                     video_id = video.get('videoId')
2990                     if not video_id:
2991                         continue
2992                     yield self._extract_video(video)
2993                     total += 1
2994                     if total == n:
2995                         return
2996             token = try_get(
2997                 slr_contents,
2998                 lambda x: x[-1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'],
2999                 compat_str)
3000             if not token:
3001                 break
3002             data['continuation'] = token
3003
3004     def _get_n_results(self, query, n):
3005         """Get a specified number of results for a query"""
3006         return self.playlist_result(self._entries(query, n), query)
3007
3008
3009 class YoutubeSearchDateIE(YoutubeSearchIE):
3010     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
3011     _SEARCH_KEY = 'ytsearchdate'
3012     IE_DESC = 'YouTube.com searches, newest videos first'
3013     _SEARCH_PARAMS = 'CAI%3D'
3014
3015
3016 r"""
3017 class YoutubeSearchURLIE(YoutubeSearchIE):
3018     IE_DESC = 'YouTube.com search URLs'
3019     IE_NAME = 'youtube:search_url'
3020     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
3021     _TESTS = [{
3022         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
3023         'playlist_mincount': 5,
3024         'info_dict': {
3025             'title': 'youtube-dl test video',
3026         }
3027     }, {
3028         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
3029         'only_matching': True,
3030     }]
3031
3032     def _real_extract(self, url):
3033         mobj = re.match(self._VALID_URL, url)
3034         query = compat_urllib_parse_unquote_plus(mobj.group('query'))
3035         webpage = self._download_webpage(url, query)
3036         return self.playlist_result(self._process_page(webpage), playlist_title=query)
3037 """
3038
3039
3040 class YoutubeFeedsInfoExtractor(YoutubeTabIE):
3041     """
3042     Base class for feed extractors
3043     Subclasses must define the _FEED_NAME property.
3044     """
3045     _LOGIN_REQUIRED = True
3046
3047     @property
3048     def IE_NAME(self):
3049         return 'youtube:%s' % self._FEED_NAME
3050
3051     def _real_initialize(self):
3052         self._login()
3053
3054     def _real_extract(self, url):
3055         return self.url_result(
3056             'https://www.youtube.com/feed/%s' % self._FEED_NAME,
3057             ie=YoutubeTabIE.ie_key())
3058
3059
3060 class YoutubeWatchLaterIE(InfoExtractor):
3061     IE_NAME = 'youtube:watchlater'
3062     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
3063     _VALID_URL = r':ytwatchlater'
3064     _TESTS = [{
3065         'url': ':ytwatchlater',
3066         'only_matching': True,
3067     }]
3068
3069     def _real_extract(self, url):
3070         return self.url_result(
3071             'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
3072
3073
3074 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
3075     IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
3076     _VALID_URL = r':ytrec(?:ommended)?'
3077     _FEED_NAME = 'recommended'
3078     _TESTS = [{
3079         'url': ':ytrec',
3080         'only_matching': True,
3081     }, {
3082         'url': ':ytrecommended',
3083         'only_matching': True,
3084     }]
3085
3086
3087 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
3088     IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
3089     _VALID_URL = r':ytsubs(?:criptions)?'
3090     _FEED_NAME = 'subscriptions'
3091     _TESTS = [{
3092         'url': ':ytsubs',
3093         'only_matching': True,
3094     }, {
3095         'url': ':ytsubscriptions',
3096         'only_matching': True,
3097     }]
3098
3099
3100 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
3101     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
3102     _VALID_URL = r':ythistory'
3103     _FEED_NAME = 'history'
3104     _TESTS = [{
3105         'url': ':ythistory',
3106         'only_matching': True,
3107     }]
3108
3109
3110 class YoutubeTruncatedURLIE(InfoExtractor):
3111     IE_NAME = 'youtube:truncated_url'
3112     IE_DESC = False  # Do not list
3113     _VALID_URL = r'''(?x)
3114         (?:https?://)?
3115         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
3116         (?:watch\?(?:
3117             feature=[a-z_]+|
3118             annotation_id=annotation_[^&]+|
3119             x-yt-cl=[0-9]+|
3120             hl=[^&]*|
3121             t=[0-9]+
3122         )?
3123         |
3124             attribution_link\?a=[^&]+
3125         )
3126         $
3127     '''
3128
3129     _TESTS = [{
3130         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
3131         'only_matching': True,
3132     }, {
3133         'url': 'https://www.youtube.com/watch?',
3134         'only_matching': True,
3135     }, {
3136         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
3137         'only_matching': True,
3138     }, {
3139         'url': 'https://www.youtube.com/watch?feature=foo',
3140         'only_matching': True,
3141     }, {
3142         'url': 'https://www.youtube.com/watch?hl=en-GB',
3143         'only_matching': True,
3144     }, {
3145         'url': 'https://www.youtube.com/watch?t=2372',
3146         'only_matching': True,
3147     }]
3148
3149     def _real_extract(self, url):
3150         raise ExtractorError(
3151             'Did you forget to quote the URL? Remember that & is a meta '
3152             'character in most shells, so you want to put the URL in quotes, '
3153             'like  youtube-dl '
3154             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
3155             ' or simply  youtube-dl BaW_jenozKc  .',
3156             expected=True)
3157
3158
3159 class YoutubeTruncatedIDIE(InfoExtractor):
3160     IE_NAME = 'youtube:truncated_id'
3161     IE_DESC = False  # Do not list
3162     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
3163
3164     _TESTS = [{
3165         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
3166         'only_matching': True,
3167     }]
3168
3169     def _real_extract(self, url):
3170         video_id = self._match_id(url)
3171         raise ExtractorError(
3172             'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
3173             expected=True)