youtube_dl/extractor/theweatherchannel.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import json
   5 import re
   6
   7 from .theplatform import ThePlatformIE
   8 from ..utils import (
   9     determine_ext,
  10     parse_duration,
  11     parse_iso8601,
  12 )
  13
  14
  15 class TheWeatherChannelIE(ThePlatformIE):
  16     _VALID_URL = r'https?://(?:www\.)?weather\.com(?P<asset_name>(?:/(?P<locale>[a-z]{2}-[A-Z]{2}))?/(?:[^/]+/)*video/(?P<id>[^/?#]+))'
  17     _TESTS = [{
  18         'url': 'https://weather.com/series/great-outdoors/video/ice-climber-is-in-for-a-shock',
  19         'md5': 'c4cbe74c9c17c5676b704b950b73dd92',
  20         'info_dict': {
  21             'id': 'cc82397e-cc3f-4d11-9390-a785add090e8',
  22             'ext': 'mp4',
  23             'title': 'Ice Climber Is In For A Shock',
  24             'description': 'md5:55606ce1378d4c72e6545e160c9d9695',
  25             'uploader': 'TWC - Digital (No Distro)',
  26             'uploader_id': '6ccd5455-16bb-46f2-9c57-ff858bb9f62c',
  27             'upload_date': '20160720',
  28             'timestamp': 1469018835,
  29         }
  30     }, {
  31         'url': 'https://weather.com/en-CA/international/videos/video/unidentified-object-falls-from-sky-in-india',
  32         'only_matching': True,
  33     }]
  34
  35     def _real_extract(self, url):
  36         asset_name, locale, display_id = re.match(self._VALID_URL, url).groups()
  37         if not locale:
  38             locale = 'en-US'
  39         video_data = list(self._download_json(
  40             'https://weather.com/api/v1/p/redux-dal', display_id, data=json.dumps([{
  41                 'name': 'getCMSAssetsUrlConfig',
  42                 'params': {
  43                     'language': locale.replace('-', '_'),
  44                     'query': {
  45                         'assetName': {
  46                             '$in': asset_name,
  47                         },
  48                     },
  49                 }
  50             }]).encode(), headers={
  51                 'Content-Type': 'application/json',
  52             })['dal']['getCMSAssetsUrlConfig'].values())[0]['data'][0]
  53         video_id = video_data['id']
  54         seo_meta = video_data.get('seometa', {})
  55         title = video_data.get('title') or seo_meta['title']
  56
  57         urls = []
  58         thumbnails = []
  59         formats = []
  60         for variant_id, variant_url in video_data.get('variants', []).items():
  61             variant_url = variant_url.strip()
  62             if not variant_url or variant_url in urls:
  63                 continue
  64             urls.append(variant_url)
  65             ext = determine_ext(variant_url)
  66             if ext == 'jpg':
  67                 thumbnails.append({
  68                     'url': variant_url,
  69                     'id': variant_id,
  70                 })
  71             elif ThePlatformIE.suitable(variant_url):
  72                 tp_formats, _ = self._extract_theplatform_smil(variant_url, video_id)
  73                 formats.extend(tp_formats)
  74             elif ext == 'm3u8':
  75                 formats.extend(self._extract_m3u8_formats(
  76                     variant_url, video_id, 'mp4', 'm3u8_native',
  77                     m3u8_id=variant_id, fatal=False))
  78             elif ext == 'f4m':
  79                 formats.extend(self._extract_f4m_formats(
  80                     variant_url, video_id, f4m_id=variant_id, fatal=False))
  81             else:
  82                 formats.append({
  83                     'url': variant_url,
  84                     'format_id': variant_id,
  85                 })
  86         self._sort_formats(formats)
  87
  88         cc_url = video_data.get('cc_url')
  89
  90         return {
  91             'id': video_id,
  92             'display_id': display_id,
  93             'title': title,
  94             'description': video_data.get('description') or seo_meta.get('description') or seo_meta.get('og:description'),
  95             'duration': parse_duration(video_data.get('duration')),
  96             'uploader': video_data.get('providername'),
  97             'uploader_id': video_data.get('providerid'),
  98             'timestamp': parse_iso8601(video_data.get('publishdate')),
  99             'subtitles': {locale[:2]: [{'url': cc_url}]} if cc_url else None,
 100             'thumbnails': thumbnails,
 101             'formats': formats,
 102         }