fix according to "https://github.com/rg3/youtube-dl/pull/15079#discussion_r158688607"
[ytdl] / youtube_dl / extractor / discoverygo.py
1 from __future__ import unicode_literals
2
3 import re
4
5 from .common import InfoExtractor
6 from ..compat import compat_str
7 from ..utils import (
8     extract_attributes,
9     ExtractorError,
10     int_or_none,
11     parse_age_limit,
12     remove_end,
13     unescapeHTML,
14 )
15
16
17 class DiscoveryGoBaseIE(InfoExtractor):
18     _VALID_URL_TEMPLATE = r'''(?x)https?://(?:www\.)?(?:
19             discovery|
20             investigationdiscovery|
21             discoverylife|
22             animalplanet|
23             ahctv|
24             destinationamerica|
25             sciencechannel|
26             tlc|
27             velocitychannel
28         )go\.com/%s(?P<id>[^/?#&]+)'''
29
30
31 class DiscoveryGoIE(DiscoveryGoBaseIE):
32     _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % r'(?:[^/]+/)+'
33     _GEO_COUNTRIES = ['US']
34     _TEST = {
35         'url': 'https://www.discoverygo.com/bering-sea-gold/reaper-madness/',
36         'info_dict': {
37             'id': '58c167d86b66d12f2addeb01',
38             'ext': 'mp4',
39             'title': 'Reaper Madness',
40             'description': 'md5:09f2c625c99afb8946ed4fb7865f6e78',
41             'duration': 2519,
42             'series': 'Bering Sea Gold',
43             'season_number': 8,
44             'episode_number': 6,
45             'age_limit': 14,
46         },
47     }
48
49     def _real_extract(self, url):
50         display_id = self._match_id(url)
51
52         webpage = self._download_webpage(url, display_id)
53
54         container = extract_attributes(
55             self._search_regex(
56                 r'(<div[^>]+class=["\']video-player-container[^>]+>)',
57                 webpage, 'video container'))
58
59         video = self._parse_json(
60             container.get('data-video') or container.get('data-json'),
61             display_id)
62
63         title = video['name']
64
65         stream = video.get('stream')
66         if not stream:
67             if video.get('authenticated') is True:
68                 raise ExtractorError(
69                     'This video is only available via cable service provider subscription that'
70                     ' is not currently supported. You may want to use --cookies.', expected=True)
71             else:
72                 raise ExtractorError('Unable to find stream')
73         STREAM_URL_SUFFIX = 'streamUrl'
74         formats = []
75         for stream_kind in ('', 'hds'):
76             suffix = STREAM_URL_SUFFIX.capitalize() if stream_kind else STREAM_URL_SUFFIX
77             stream_url = stream.get('%s%s' % (stream_kind, suffix))
78             if not stream_url:
79                 continue
80             if stream_kind == '':
81                 formats.extend(self._extract_m3u8_formats(
82                     stream_url, display_id, 'mp4', entry_protocol='m3u8_native',
83                     m3u8_id='hls', fatal=False))
84             elif stream_kind == 'hds':
85                 formats.extend(self._extract_f4m_formats(
86                     stream_url, display_id, f4m_id=stream_kind, fatal=False))
87         self._sort_formats(formats)
88
89         video_id = video.get('id') or display_id
90         description = video.get('description', {}).get('detailed')
91         duration = int_or_none(video.get('duration'))
92
93         series = video.get('show', {}).get('name')
94         season_number = int_or_none(video.get('season', {}).get('number'))
95         episode_number = int_or_none(video.get('episodeNumber'))
96
97         tags = video.get('tags')
98         age_limit = parse_age_limit(video.get('parental', {}).get('rating'))
99
100         subtitles = {}
101         captions = stream.get('captions')
102         if isinstance(captions, list):
103             for caption in captions:
104                 subtitle_url = caption.get('fileUrl')
105                 if (not subtitle_url or not isinstance(subtitle_url, compat_str) or
106                         not subtitle_url.startswith('http')):
107                     continue
108                 lang = caption.get('fileLang', 'en')
109                 subtitles.setdefault(lang, []).append({'url': subtitle_url})
110
111         return {
112             'id': video_id,
113             'display_id': display_id,
114             'title': title,
115             'description': description,
116             'duration': duration,
117             'series': series,
118             'season_number': season_number,
119             'episode_number': episode_number,
120             'tags': tags,
121             'age_limit': age_limit,
122             'formats': formats,
123             'subtitles': subtitles,
124         }
125
126
127 class DiscoveryGoPlaylistIE(DiscoveryGoBaseIE):
128     _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % ''
129     _TEST = {
130         'url': 'https://www.discoverygo.com/bering-sea-gold/',
131         'info_dict': {
132             'id': 'bering-sea-gold',
133             'title': 'Bering Sea Gold',
134             'description': 'md5:cc5c6489835949043c0cc3ad66c2fa0e',
135         },
136         'playlist_mincount': 6,
137     }
138
139     @classmethod
140     def suitable(cls, url):
141         return False if DiscoveryGoIE.suitable(url) else super(
142             DiscoveryGoPlaylistIE, cls).suitable(url)
143
144     def _real_extract(self, url):
145         display_id = self._match_id(url)
146
147         webpage = self._download_webpage(url, display_id)
148
149         entries = []
150         for mobj in re.finditer(r'data-json=(["\'])(?P<json>{.+?})\1', webpage):
151             data = self._parse_json(
152                 mobj.group('json'), display_id,
153                 transform_source=unescapeHTML, fatal=False)
154             if not isinstance(data, dict) or data.get('type') != 'episode':
155                 continue
156             episode_url = data.get('socialUrl')
157             if not episode_url:
158                 continue
159             entries.append(self.url_result(
160                 episode_url, ie=DiscoveryGoIE.ie_key(),
161                 video_id=data.get('id')))
162
163         return self.playlist_result(
164             entries, display_id,
165             remove_end(self._og_search_title(
166                 webpage, fatal=False), ' | Discovery GO'),
167             self._og_search_description(webpage))