[periscope] pass referer to HLS requests(closes #29419)
[ytdl] / youtube_dl / extractor / periscope.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..utils import (
8     int_or_none,
9     parse_iso8601,
10     unescapeHTML,
11 )
12
13
14 class PeriscopeBaseIE(InfoExtractor):
15     _M3U8_HEADERS = {
16         'Referer': 'https://www.periscope.tv/'
17     }
18
19     def _call_api(self, method, query, item_id):
20         return self._download_json(
21             'https://api.periscope.tv/api/v2/%s' % method,
22             item_id, query=query)
23
24     def _parse_broadcast_data(self, broadcast, video_id):
25         title = broadcast.get('status') or 'Periscope Broadcast'
26         uploader = broadcast.get('user_display_name') or broadcast.get('username')
27         title = '%s - %s' % (uploader, title) if uploader else title
28         is_live = broadcast.get('state').lower() == 'running'
29
30         thumbnails = [{
31             'url': broadcast[image],
32         } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
33
34         return {
35             'id': broadcast.get('id') or video_id,
36             'title': self._live_title(title) if is_live else title,
37             'timestamp': parse_iso8601(broadcast.get('created_at')),
38             'uploader': uploader,
39             'uploader_id': broadcast.get('user_id') or broadcast.get('username'),
40             'thumbnails': thumbnails,
41             'view_count': int_or_none(broadcast.get('total_watched')),
42             'tags': broadcast.get('tags'),
43             'is_live': is_live,
44         }
45
46     @staticmethod
47     def _extract_common_format_info(broadcast):
48         return broadcast.get('state').lower(), int_or_none(broadcast.get('width')), int_or_none(broadcast.get('height'))
49
50     @staticmethod
51     def _add_width_and_height(f, width, height):
52         for key, val in (('width', width), ('height', height)):
53             if not f.get(key):
54                 f[key] = val
55
56     def _extract_pscp_m3u8_formats(self, m3u8_url, video_id, format_id, state, width, height, fatal=True):
57         m3u8_formats = self._extract_m3u8_formats(
58             m3u8_url, video_id, 'mp4',
59             entry_protocol='m3u8_native'
60             if state in ('ended', 'timed_out') else 'm3u8',
61             m3u8_id=format_id, fatal=fatal, headers=self._M3U8_HEADERS)
62         if len(m3u8_formats) == 1:
63             self._add_width_and_height(m3u8_formats[0], width, height)
64         for f in m3u8_formats:
65             f.setdefault('http_headers', {}).update(self._M3U8_HEADERS)
66         return m3u8_formats
67
68
69 class PeriscopeIE(PeriscopeBaseIE):
70     IE_DESC = 'Periscope'
71     IE_NAME = 'periscope'
72     _VALID_URL = r'https?://(?:www\.)?(?:periscope|pscp)\.tv/[^/]+/(?P<id>[^/?#]+)'
73     # Alive example URLs can be found here https://www.periscope.tv/
74     _TESTS = [{
75         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
76         'md5': '65b57957972e503fcbbaeed8f4fa04ca',
77         'info_dict': {
78             'id': '56102209',
79             'ext': 'mp4',
80             'title': 'Bec Boop - ๐Ÿš โœˆ๏ธ๐Ÿ‡ฌ๐Ÿ‡ง Fly above #London in Emirates Air Line cable car at night ๐Ÿ‡ฌ๐Ÿ‡งโœˆ๏ธ๐Ÿš  #BoopScope ๐ŸŽ€๐Ÿ’—',
81             'timestamp': 1438978559,
82             'upload_date': '20150807',
83             'uploader': 'Bec Boop',
84             'uploader_id': '1465763',
85         },
86         'skip': 'Expires in 24 hours',
87     }, {
88         'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
89         'only_matching': True,
90     }, {
91         'url': 'https://www.periscope.tv/bastaakanoggano/1OdKrlkZZjOJX',
92         'only_matching': True,
93     }, {
94         'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
95         'only_matching': True,
96     }]
97
98     @staticmethod
99     def _extract_url(webpage):
100         mobj = re.search(
101             r'<iframe[^>]+src=([\'"])(?P<url>(?:https?:)?//(?:www\.)?(?:periscope|pscp)\.tv/(?:(?!\1).)+)\1', webpage)
102         if mobj:
103             return mobj.group('url')
104
105     def _real_extract(self, url):
106         token = self._match_id(url)
107
108         stream = self._call_api(
109             'accessVideoPublic', {'broadcast_id': token}, token)
110
111         broadcast = stream['broadcast']
112         info = self._parse_broadcast_data(broadcast, token)
113
114         state = broadcast.get('state').lower()
115         width = int_or_none(broadcast.get('width'))
116         height = int_or_none(broadcast.get('height'))
117
118         def add_width_and_height(f):
119             for key, val in (('width', width), ('height', height)):
120                 if not f.get(key):
121                     f[key] = val
122
123         video_urls = set()
124         formats = []
125         for format_id in ('replay', 'rtmp', 'hls', 'https_hls', 'lhls', 'lhlsweb'):
126             video_url = stream.get(format_id + '_url')
127             if not video_url or video_url in video_urls:
128                 continue
129             video_urls.add(video_url)
130             if format_id != 'rtmp':
131                 m3u8_formats = self._extract_pscp_m3u8_formats(
132                     video_url, token, format_id, state, width, height, False)
133                 formats.extend(m3u8_formats)
134                 continue
135             rtmp_format = {
136                 'url': video_url,
137                 'ext': 'flv' if format_id == 'rtmp' else 'mp4',
138             }
139             self._add_width_and_height(rtmp_format)
140             formats.append(rtmp_format)
141         self._sort_formats(formats)
142
143         info['formats'] = formats
144         return info
145
146
147 class PeriscopeUserIE(PeriscopeBaseIE):
148     _VALID_URL = r'https?://(?:www\.)?(?:periscope|pscp)\.tv/(?P<id>[^/]+)/?$'
149     IE_DESC = 'Periscope user videos'
150     IE_NAME = 'periscope:user'
151
152     _TEST = {
153         'url': 'https://www.periscope.tv/LularoeHusbandMike/',
154         'info_dict': {
155             'id': 'LularoeHusbandMike',
156             'title': 'LULAROE HUSBAND MIKE',
157             'description': 'md5:6cf4ec8047768098da58e446e82c82f0',
158         },
159         # Periscope only shows videos in the last 24 hours, so it's possible to
160         # get 0 videos
161         'playlist_mincount': 0,
162     }
163
164     def _real_extract(self, url):
165         user_name = self._match_id(url)
166
167         webpage = self._download_webpage(url, user_name)
168
169         data_store = self._parse_json(
170             unescapeHTML(self._search_regex(
171                 r'data-store=(["\'])(?P<data>.+?)\1',
172                 webpage, 'data store', default='{}', group='data')),
173             user_name)
174
175         user = list(data_store['UserCache']['users'].values())[0]['user']
176         user_id = user['id']
177         session_id = data_store['SessionToken']['public']['broadcastHistory']['token']['session_id']
178
179         broadcasts = self._call_api(
180             'getUserBroadcastsPublic',
181             {'user_id': user_id, 'session_id': session_id},
182             user_name)['broadcasts']
183
184         broadcast_ids = [
185             broadcast['id'] for broadcast in broadcasts if broadcast.get('id')]
186
187         title = user.get('display_name') or user.get('username') or user_name
188         description = user.get('description')
189
190         entries = [
191             self.url_result(
192                 'https://www.periscope.tv/%s/%s' % (user_name, broadcast_id))
193             for broadcast_id in broadcast_ids]
194
195         return self.playlist_result(entries, user_id, title, description)