[orf:tvthek] Fix thumbnails extraction (closes #29217)
[ytdl] / youtube_dl / extractor / orf.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import compat_str
8 from ..utils import (
9     clean_html,
10     determine_ext,
11     float_or_none,
12     HEADRequest,
13     int_or_none,
14     orderedSet,
15     remove_end,
16     str_or_none,
17     strip_jsonp,
18     unescapeHTML,
19     unified_strdate,
20     url_or_none,
21 )
22
23
24 class ORFTVthekIE(InfoExtractor):
25     IE_NAME = 'orf:tvthek'
26     IE_DESC = 'ORF TVthek'
27     _VALID_URL = r'https?://tvthek\.orf\.at/(?:[^/]+/)+(?P<id>\d+)'
28
29     _TESTS = [{
30         'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
31         'playlist': [{
32             'md5': '2942210346ed779588f428a92db88712',
33             'info_dict': {
34                 'id': '8896777',
35                 'ext': 'mp4',
36                 'title': 'Aufgetischt: Mit der Steirischen Tafelrunde',
37                 'description': 'md5:c1272f0245537812d4e36419c207b67d',
38                 'duration': 2668,
39                 'upload_date': '20141208',
40             },
41         }],
42         'skip': 'Blocked outside of Austria / Germany',
43     }, {
44         'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
45         'info_dict': {
46             'id': '7982259',
47             'ext': 'mp4',
48             'title': 'Best of Ingrid Thurnher',
49             'upload_date': '20140527',
50             'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im Jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
51         },
52         'params': {
53             'skip_download': True,  # rtsp downloads
54         },
55         'skip': 'Blocked outside of Austria / Germany',
56     }, {
57         'url': 'http://tvthek.orf.at/topic/Fluechtlingskrise/10463081/Heimat-Fremde-Heimat/13879132/Senioren-betreuen-Migrantenkinder/13879141',
58         'only_matching': True,
59     }, {
60         'url': 'http://tvthek.orf.at/profile/Universum/35429',
61         'only_matching': True,
62     }]
63
64     def _real_extract(self, url):
65         playlist_id = self._match_id(url)
66         webpage = self._download_webpage(url, playlist_id)
67
68         data_jsb = self._parse_json(
69             self._search_regex(
70                 r'<div[^>]+class=(["\']).*?VideoPlaylist.*?\1[^>]+data-jsb=(["\'])(?P<json>.+?)\2',
71                 webpage, 'playlist', group='json'),
72             playlist_id, transform_source=unescapeHTML)['playlist']['videos']
73
74         entries = []
75         for sd in data_jsb:
76             video_id, title = sd.get('id'), sd.get('title')
77             if not video_id or not title:
78                 continue
79             video_id = compat_str(video_id)
80             formats = []
81             for fd in sd['sources']:
82                 src = url_or_none(fd.get('src'))
83                 if not src:
84                     continue
85                 format_id_list = []
86                 for key in ('delivery', 'quality', 'quality_string'):
87                     value = fd.get(key)
88                     if value:
89                         format_id_list.append(value)
90                 format_id = '-'.join(format_id_list)
91                 ext = determine_ext(src)
92                 if ext == 'm3u8':
93                     m3u8_formats = self._extract_m3u8_formats(
94                         src, video_id, 'mp4', m3u8_id=format_id, fatal=False)
95                     if any('/geoprotection' in f['url'] for f in m3u8_formats):
96                         self.raise_geo_restricted()
97                     formats.extend(m3u8_formats)
98                 elif ext == 'f4m':
99                     formats.extend(self._extract_f4m_formats(
100                         src, video_id, f4m_id=format_id, fatal=False))
101                 else:
102                     formats.append({
103                         'format_id': format_id,
104                         'url': src,
105                         'protocol': fd.get('protocol'),
106                     })
107
108             # Check for geoblocking.
109             # There is a property is_geoprotection, but that's always false
110             geo_str = sd.get('geoprotection_string')
111             if geo_str:
112                 try:
113                     http_url = next(
114                         f['url']
115                         for f in formats
116                         if re.match(r'^https?://.*\.mp4$', f['url']))
117                 except StopIteration:
118                     pass
119                 else:
120                     req = HEADRequest(http_url)
121                     self._request_webpage(
122                         req, video_id,
123                         note='Testing for geoblocking',
124                         errnote=((
125                             'This video seems to be blocked outside of %s. '
126                             'You may want to try the streaming-* formats.')
127                             % geo_str),
128                         fatal=False)
129
130             self._check_formats(formats, video_id)
131             self._sort_formats(formats)
132
133             subtitles = {}
134             for sub in sd.get('subtitles', []):
135                 sub_src = sub.get('src')
136                 if not sub_src:
137                     continue
138                 subtitles.setdefault(sub.get('lang', 'de-AT'), []).append({
139                     'url': sub_src,
140                 })
141
142             upload_date = unified_strdate(sd.get('created_date'))
143
144             thumbnails = []
145             preview = sd.get('preview_image_url')
146             if preview:
147                 thumbnails.append({
148                     'id': 'preview',
149                     'url': preview,
150                     'preference': 0,
151                 })
152             image = sd.get('image_full_url')
153             if not image and len(data_jsb) == 1:
154                 image = self._og_search_thumbnail(webpage)
155             if image:
156                 thumbnails.append({
157                     'id': 'full',
158                     'url': image,
159                     'preference': 1,
160                 })
161
162             entries.append({
163                 '_type': 'video',
164                 'id': video_id,
165                 'title': title,
166                 'formats': formats,
167                 'subtitles': subtitles,
168                 'description': sd.get('description'),
169                 'duration': int_or_none(sd.get('duration_in_seconds')),
170                 'upload_date': upload_date,
171                 'thumbnails': thumbnails,
172             })
173
174         return {
175             '_type': 'playlist',
176             'entries': entries,
177             'id': playlist_id,
178         }
179
180
181 class ORFRadioIE(InfoExtractor):
182     def _real_extract(self, url):
183         mobj = re.match(self._VALID_URL, url)
184         show_date = mobj.group('date')
185         show_id = mobj.group('show')
186
187         data = self._download_json(
188             'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s'
189             % (self._API_STATION, show_id, show_date), show_id)
190
191         entries = []
192         for info in data['streams']:
193             loop_stream_id = str_or_none(info.get('loopStreamId'))
194             if not loop_stream_id:
195                 continue
196             title = str_or_none(data.get('title'))
197             if not title:
198                 continue
199             start = int_or_none(info.get('start'), scale=1000)
200             end = int_or_none(info.get('end'), scale=1000)
201             duration = end - start if end and start else None
202             entries.append({
203                 'id': loop_stream_id.replace('.mp3', ''),
204                 'url': 'https://loopstream01.apa.at/?channel=%s&id=%s' % (self._LOOP_STATION, loop_stream_id),
205                 'title': title,
206                 'description': clean_html(data.get('subtitle')),
207                 'duration': duration,
208                 'timestamp': start,
209                 'ext': 'mp3',
210                 'series': data.get('programTitle'),
211             })
212
213         return {
214             '_type': 'playlist',
215             'id': show_id,
216             'title': data.get('title'),
217             'description': clean_html(data.get('subtitle')),
218             'entries': entries,
219         }
220
221
222 class ORFFM4IE(ORFRadioIE):
223     IE_NAME = 'orf:fm4'
224     IE_DESC = 'radio FM4'
225     _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>4\w+)'
226     _API_STATION = 'fm4'
227     _LOOP_STATION = 'fm4'
228
229     _TEST = {
230         'url': 'http://fm4.orf.at/player/20170107/4CC',
231         'md5': '2b0be47375432a7ef104453432a19212',
232         'info_dict': {
233             'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
234             'ext': 'mp3',
235             'title': 'Solid Steel Radioshow',
236             'description': 'Die Mixshow von Coldcut und Ninja Tune.',
237             'duration': 3599,
238             'timestamp': 1483819257,
239             'upload_date': '20170107',
240         },
241         'skip': 'Shows from ORF radios are only available for 7 days.',
242         'only_matching': True,
243     }
244
245
246 class ORFNOEIE(ORFRadioIE):
247     IE_NAME = 'orf:noe'
248     IE_DESC = 'Radio Niederösterreich'
249     _VALID_URL = r'https?://(?P<station>noe)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
250     _API_STATION = 'noe'
251     _LOOP_STATION = 'oe2n'
252
253     _TEST = {
254         'url': 'https://noe.orf.at/player/20200423/NGM',
255         'only_matching': True,
256     }
257
258
259 class ORFWIEIE(ORFRadioIE):
260     IE_NAME = 'orf:wien'
261     IE_DESC = 'Radio Wien'
262     _VALID_URL = r'https?://(?P<station>wien)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
263     _API_STATION = 'wie'
264     _LOOP_STATION = 'oe2w'
265
266     _TEST = {
267         'url': 'https://wien.orf.at/player/20200423/WGUM',
268         'only_matching': True,
269     }
270
271
272 class ORFBGLIE(ORFRadioIE):
273     IE_NAME = 'orf:burgenland'
274     IE_DESC = 'Radio Burgenland'
275     _VALID_URL = r'https?://(?P<station>burgenland)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
276     _API_STATION = 'bgl'
277     _LOOP_STATION = 'oe2b'
278
279     _TEST = {
280         'url': 'https://burgenland.orf.at/player/20200423/BGM',
281         'only_matching': True,
282     }
283
284
285 class ORFOOEIE(ORFRadioIE):
286     IE_NAME = 'orf:oberoesterreich'
287     IE_DESC = 'Radio Oberösterreich'
288     _VALID_URL = r'https?://(?P<station>ooe)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
289     _API_STATION = 'ooe'
290     _LOOP_STATION = 'oe2o'
291
292     _TEST = {
293         'url': 'https://ooe.orf.at/player/20200423/OGMO',
294         'only_matching': True,
295     }
296
297
298 class ORFSTMIE(ORFRadioIE):
299     IE_NAME = 'orf:steiermark'
300     IE_DESC = 'Radio Steiermark'
301     _VALID_URL = r'https?://(?P<station>steiermark)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
302     _API_STATION = 'stm'
303     _LOOP_STATION = 'oe2st'
304
305     _TEST = {
306         'url': 'https://steiermark.orf.at/player/20200423/STGMS',
307         'only_matching': True,
308     }
309
310
311 class ORFKTNIE(ORFRadioIE):
312     IE_NAME = 'orf:kaernten'
313     IE_DESC = 'Radio Kärnten'
314     _VALID_URL = r'https?://(?P<station>kaernten)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
315     _API_STATION = 'ktn'
316     _LOOP_STATION = 'oe2k'
317
318     _TEST = {
319         'url': 'https://kaernten.orf.at/player/20200423/KGUMO',
320         'only_matching': True,
321     }
322
323
324 class ORFSBGIE(ORFRadioIE):
325     IE_NAME = 'orf:salzburg'
326     IE_DESC = 'Radio Salzburg'
327     _VALID_URL = r'https?://(?P<station>salzburg)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
328     _API_STATION = 'sbg'
329     _LOOP_STATION = 'oe2s'
330
331     _TEST = {
332         'url': 'https://salzburg.orf.at/player/20200423/SGUM',
333         'only_matching': True,
334     }
335
336
337 class ORFTIRIE(ORFRadioIE):
338     IE_NAME = 'orf:tirol'
339     IE_DESC = 'Radio Tirol'
340     _VALID_URL = r'https?://(?P<station>tirol)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
341     _API_STATION = 'tir'
342     _LOOP_STATION = 'oe2t'
343
344     _TEST = {
345         'url': 'https://tirol.orf.at/player/20200423/TGUMO',
346         'only_matching': True,
347     }
348
349
350 class ORFVBGIE(ORFRadioIE):
351     IE_NAME = 'orf:vorarlberg'
352     IE_DESC = 'Radio Vorarlberg'
353     _VALID_URL = r'https?://(?P<station>vorarlberg)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
354     _API_STATION = 'vbg'
355     _LOOP_STATION = 'oe2v'
356
357     _TEST = {
358         'url': 'https://vorarlberg.orf.at/player/20200423/VGUM',
359         'only_matching': True,
360     }
361
362
363 class ORFOE3IE(ORFRadioIE):
364     IE_NAME = 'orf:oe3'
365     IE_DESC = 'Radio Österreich 3'
366     _VALID_URL = r'https?://(?P<station>oe3)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
367     _API_STATION = 'oe3'
368     _LOOP_STATION = 'oe3'
369
370     _TEST = {
371         'url': 'https://oe3.orf.at/player/20200424/3WEK',
372         'only_matching': True,
373     }
374
375
376 class ORFOE1IE(ORFRadioIE):
377     IE_NAME = 'orf:oe1'
378     IE_DESC = 'Radio Österreich 1'
379     _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
380     _API_STATION = 'oe1'
381     _LOOP_STATION = 'oe1'
382
383     _TEST = {
384         'url': 'http://oe1.orf.at/player/20170108/456544',
385         'md5': '34d8a6e67ea888293741c86a099b745b',
386         'info_dict': {
387             'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
388             'ext': 'mp3',
389             'title': 'Morgenjournal',
390             'duration': 609,
391             'timestamp': 1483858796,
392             'upload_date': '20170108',
393         },
394         'skip': 'Shows from ORF radios are only available for 7 days.'
395     }
396
397
398 class ORFIPTVIE(InfoExtractor):
399     IE_NAME = 'orf:iptv'
400     IE_DESC = 'iptv.ORF.at'
401     _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
402
403     _TEST = {
404         'url': 'http://iptv.orf.at/stories/2275236/',
405         'md5': 'c8b22af4718a4b4af58342529453e3e5',
406         'info_dict': {
407             'id': '350612',
408             'ext': 'flv',
409             'title': 'Weitere Evakuierungen um Vulkan Calbuco',
410             'description': 'md5:d689c959bdbcf04efeddedbf2299d633',
411             'duration': 68.197,
412             'thumbnail': r're:^https?://.*\.jpg$',
413             'upload_date': '20150425',
414         },
415     }
416
417     def _real_extract(self, url):
418         story_id = self._match_id(url)
419
420         webpage = self._download_webpage(
421             'http://iptv.orf.at/stories/%s' % story_id, story_id)
422
423         video_id = self._search_regex(
424             r'data-video(?:id)?="(\d+)"', webpage, 'video id')
425
426         data = self._download_json(
427             'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
428             video_id)[0]
429
430         duration = float_or_none(data['duration'], 1000)
431
432         video = data['sources']['default']
433         load_balancer_url = video['loadBalancerUrl']
434         abr = int_or_none(video.get('audioBitrate'))
435         vbr = int_or_none(video.get('bitrate'))
436         fps = int_or_none(video.get('videoFps'))
437         width = int_or_none(video.get('videoWidth'))
438         height = int_or_none(video.get('videoHeight'))
439         thumbnail = video.get('preview')
440
441         rendition = self._download_json(
442             load_balancer_url, video_id, transform_source=strip_jsonp)
443
444         f = {
445             'abr': abr,
446             'vbr': vbr,
447             'fps': fps,
448             'width': width,
449             'height': height,
450         }
451
452         formats = []
453         for format_id, format_url in rendition['redirect'].items():
454             if format_id == 'rtmp':
455                 ff = f.copy()
456                 ff.update({
457                     'url': format_url,
458                     'format_id': format_id,
459                 })
460                 formats.append(ff)
461             elif determine_ext(format_url) == 'f4m':
462                 formats.extend(self._extract_f4m_formats(
463                     format_url, video_id, f4m_id=format_id))
464             elif determine_ext(format_url) == 'm3u8':
465                 formats.extend(self._extract_m3u8_formats(
466                     format_url, video_id, 'mp4', m3u8_id=format_id))
467             else:
468                 continue
469         self._sort_formats(formats)
470
471         title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at')
472         description = self._og_search_description(webpage)
473         upload_date = unified_strdate(self._html_search_meta(
474             'dc.date', webpage, 'upload date'))
475
476         return {
477             'id': video_id,
478             'title': title,
479             'description': description,
480             'duration': duration,
481             'thumbnail': thumbnail,
482             'upload_date': upload_date,
483             'formats': formats,
484         }
485
486
487 class ORFFM4StoryIE(InfoExtractor):
488     IE_NAME = 'orf:fm4:story'
489     IE_DESC = 'fm4.orf.at stories'
490     _VALID_URL = r'https?://fm4\.orf\.at/stories/(?P<id>\d+)'
491
492     _TEST = {
493         'url': 'http://fm4.orf.at/stories/2865738/',
494         'playlist': [{
495             'md5': 'e1c2c706c45c7b34cf478bbf409907ca',
496             'info_dict': {
497                 'id': '547792',
498                 'ext': 'flv',
499                 'title': 'Manu Delago und Inner Tongue live',
500                 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
501                 'duration': 1748.52,
502                 'thumbnail': r're:^https?://.*\.jpg$',
503                 'upload_date': '20170913',
504             },
505         }, {
506             'md5': 'c6dd2179731f86f4f55a7b49899d515f',
507             'info_dict': {
508                 'id': '547798',
509                 'ext': 'flv',
510                 'title': 'Manu Delago und Inner Tongue live (2)',
511                 'duration': 1504.08,
512                 'thumbnail': r're:^https?://.*\.jpg$',
513                 'upload_date': '20170913',
514                 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
515             },
516         }],
517     }
518
519     def _real_extract(self, url):
520         story_id = self._match_id(url)
521         webpage = self._download_webpage(url, story_id)
522
523         entries = []
524         all_ids = orderedSet(re.findall(r'data-video(?:id)?="(\d+)"', webpage))
525         for idx, video_id in enumerate(all_ids):
526             data = self._download_json(
527                 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
528                 video_id)[0]
529
530             duration = float_or_none(data['duration'], 1000)
531
532             video = data['sources']['q8c']
533             load_balancer_url = video['loadBalancerUrl']
534             abr = int_or_none(video.get('audioBitrate'))
535             vbr = int_or_none(video.get('bitrate'))
536             fps = int_or_none(video.get('videoFps'))
537             width = int_or_none(video.get('videoWidth'))
538             height = int_or_none(video.get('videoHeight'))
539             thumbnail = video.get('preview')
540
541             rendition = self._download_json(
542                 load_balancer_url, video_id, transform_source=strip_jsonp)
543
544             f = {
545                 'abr': abr,
546                 'vbr': vbr,
547                 'fps': fps,
548                 'width': width,
549                 'height': height,
550             }
551
552             formats = []
553             for format_id, format_url in rendition['redirect'].items():
554                 if format_id == 'rtmp':
555                     ff = f.copy()
556                     ff.update({
557                         'url': format_url,
558                         'format_id': format_id,
559                     })
560                     formats.append(ff)
561                 elif determine_ext(format_url) == 'f4m':
562                     formats.extend(self._extract_f4m_formats(
563                         format_url, video_id, f4m_id=format_id))
564                 elif determine_ext(format_url) == 'm3u8':
565                     formats.extend(self._extract_m3u8_formats(
566                         format_url, video_id, 'mp4', m3u8_id=format_id))
567                 else:
568                     continue
569             self._sort_formats(formats)
570
571             title = remove_end(self._og_search_title(webpage), ' - fm4.ORF.at')
572             if idx >= 1:
573                 # Titles are duplicates, make them unique
574                 title += ' (' + str(idx + 1) + ')'
575             description = self._og_search_description(webpage)
576             upload_date = unified_strdate(self._html_search_meta(
577                 'dc.date', webpage, 'upload date'))
578
579             entries.append({
580                 'id': video_id,
581                 'title': title,
582                 'description': description,
583                 'duration': duration,
584                 'thumbnail': thumbnail,
585                 'upload_date': upload_date,
586                 'formats': formats,
587             })
588
589         return self.playlist_result(entries)