db936bfb88d4bd0a05be1cc6fd7d1b18ce3a72b6
[ytdl] / test / test_YoutubeDL.py
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 from __future__ import unicode_literals
5
6 # Allow direct execution
7 import os
8 import sys
9 import unittest
10 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
11
12 import copy
13
14 from test.helper import FakeYDL, assertRegexpMatches
15 from youtube_dl import YoutubeDL
16 from youtube_dl.compat import compat_str, compat_urllib_error
17 from youtube_dl.extractor import YoutubeIE
18 from youtube_dl.extractor.common import InfoExtractor
19 from youtube_dl.postprocessor.common import PostProcessor
20 from youtube_dl.utils import ExtractorError, match_filter_func
21
22 TEST_URL = 'http://localhost/sample.mp4'
23
24
25 class YDL(FakeYDL):
26     def __init__(self, *args, **kwargs):
27         super(YDL, self).__init__(*args, **kwargs)
28         self.downloaded_info_dicts = []
29         self.msgs = []
30
31     def process_info(self, info_dict):
32         self.downloaded_info_dicts.append(info_dict)
33
34     def to_screen(self, msg):
35         self.msgs.append(msg)
36
37
38 def _make_result(formats, **kwargs):
39     res = {
40         'formats': formats,
41         'id': 'testid',
42         'title': 'testttitle',
43         'extractor': 'testex',
44         'extractor_key': 'TestEx',
45     }
46     res.update(**kwargs)
47     return res
48
49
50 class TestFormatSelection(unittest.TestCase):
51     def test_prefer_free_formats(self):
52         # Same resolution => download webm
53         ydl = YDL()
54         ydl.params['prefer_free_formats'] = True
55         formats = [
56             {'ext': 'webm', 'height': 460, 'url': TEST_URL},
57             {'ext': 'mp4', 'height': 460, 'url': TEST_URL},
58         ]
59         info_dict = _make_result(formats)
60         yie = YoutubeIE(ydl)
61         yie._sort_formats(info_dict['formats'])
62         ydl.process_ie_result(info_dict)
63         downloaded = ydl.downloaded_info_dicts[0]
64         self.assertEqual(downloaded['ext'], 'webm')
65
66         # Different resolution => download best quality (mp4)
67         ydl = YDL()
68         ydl.params['prefer_free_formats'] = True
69         formats = [
70             {'ext': 'webm', 'height': 720, 'url': TEST_URL},
71             {'ext': 'mp4', 'height': 1080, 'url': TEST_URL},
72         ]
73         info_dict['formats'] = formats
74         yie = YoutubeIE(ydl)
75         yie._sort_formats(info_dict['formats'])
76         ydl.process_ie_result(info_dict)
77         downloaded = ydl.downloaded_info_dicts[0]
78         self.assertEqual(downloaded['ext'], 'mp4')
79
80         # No prefer_free_formats => prefer mp4 and flv for greater compatibility
81         ydl = YDL()
82         ydl.params['prefer_free_formats'] = False
83         formats = [
84             {'ext': 'webm', 'height': 720, 'url': TEST_URL},
85             {'ext': 'mp4', 'height': 720, 'url': TEST_URL},
86             {'ext': 'flv', 'height': 720, 'url': TEST_URL},
87         ]
88         info_dict['formats'] = formats
89         yie = YoutubeIE(ydl)
90         yie._sort_formats(info_dict['formats'])
91         ydl.process_ie_result(info_dict)
92         downloaded = ydl.downloaded_info_dicts[0]
93         self.assertEqual(downloaded['ext'], 'mp4')
94
95         ydl = YDL()
96         ydl.params['prefer_free_formats'] = False
97         formats = [
98             {'ext': 'flv', 'height': 720, 'url': TEST_URL},
99             {'ext': 'webm', 'height': 720, 'url': TEST_URL},
100         ]
101         info_dict['formats'] = formats
102         yie = YoutubeIE(ydl)
103         yie._sort_formats(info_dict['formats'])
104         ydl.process_ie_result(info_dict)
105         downloaded = ydl.downloaded_info_dicts[0]
106         self.assertEqual(downloaded['ext'], 'flv')
107
108     def test_format_selection(self):
109         formats = [
110             {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
111             {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
112             {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
113             {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
114             {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
115         ]
116         info_dict = _make_result(formats)
117
118         ydl = YDL({'format': '20/47'})
119         ydl.process_ie_result(info_dict.copy())
120         downloaded = ydl.downloaded_info_dicts[0]
121         self.assertEqual(downloaded['format_id'], '47')
122
123         ydl = YDL({'format': '20/71/worst'})
124         ydl.process_ie_result(info_dict.copy())
125         downloaded = ydl.downloaded_info_dicts[0]
126         self.assertEqual(downloaded['format_id'], '35')
127
128         ydl = YDL()
129         ydl.process_ie_result(info_dict.copy())
130         downloaded = ydl.downloaded_info_dicts[0]
131         self.assertEqual(downloaded['format_id'], '2')
132
133         ydl = YDL({'format': 'webm/mp4'})
134         ydl.process_ie_result(info_dict.copy())
135         downloaded = ydl.downloaded_info_dicts[0]
136         self.assertEqual(downloaded['format_id'], '47')
137
138         ydl = YDL({'format': '3gp/40/mp4'})
139         ydl.process_ie_result(info_dict.copy())
140         downloaded = ydl.downloaded_info_dicts[0]
141         self.assertEqual(downloaded['format_id'], '35')
142
143         ydl = YDL({'format': 'example-with-dashes'})
144         ydl.process_ie_result(info_dict.copy())
145         downloaded = ydl.downloaded_info_dicts[0]
146         self.assertEqual(downloaded['format_id'], 'example-with-dashes')
147
148     def test_format_selection_audio(self):
149         formats = [
150             {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
151             {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
152             {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL},
153             {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL},
154         ]
155         info_dict = _make_result(formats)
156
157         ydl = YDL({'format': 'bestaudio'})
158         ydl.process_ie_result(info_dict.copy())
159         downloaded = ydl.downloaded_info_dicts[0]
160         self.assertEqual(downloaded['format_id'], 'audio-high')
161
162         ydl = YDL({'format': 'worstaudio'})
163         ydl.process_ie_result(info_dict.copy())
164         downloaded = ydl.downloaded_info_dicts[0]
165         self.assertEqual(downloaded['format_id'], 'audio-low')
166
167         formats = [
168             {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
169             {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL},
170         ]
171         info_dict = _make_result(formats)
172
173         ydl = YDL({'format': 'bestaudio/worstaudio/best'})
174         ydl.process_ie_result(info_dict.copy())
175         downloaded = ydl.downloaded_info_dicts[0]
176         self.assertEqual(downloaded['format_id'], 'vid-high')
177
178     def test_format_selection_audio_exts(self):
179         formats = [
180             {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
181             {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
182             {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
183             {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
184             {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
185         ]
186
187         info_dict = _make_result(formats)
188         ydl = YDL({'format': 'best'})
189         ie = YoutubeIE(ydl)
190         ie._sort_formats(info_dict['formats'])
191         ydl.process_ie_result(copy.deepcopy(info_dict))
192         downloaded = ydl.downloaded_info_dicts[0]
193         self.assertEqual(downloaded['format_id'], 'aac-64')
194
195         ydl = YDL({'format': 'mp3'})
196         ie = YoutubeIE(ydl)
197         ie._sort_formats(info_dict['formats'])
198         ydl.process_ie_result(copy.deepcopy(info_dict))
199         downloaded = ydl.downloaded_info_dicts[0]
200         self.assertEqual(downloaded['format_id'], 'mp3-64')
201
202         ydl = YDL({'prefer_free_formats': True})
203         ie = YoutubeIE(ydl)
204         ie._sort_formats(info_dict['formats'])
205         ydl.process_ie_result(copy.deepcopy(info_dict))
206         downloaded = ydl.downloaded_info_dicts[0]
207         self.assertEqual(downloaded['format_id'], 'ogg-64')
208
209     def test_format_selection_video(self):
210         formats = [
211             {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL},
212             {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL},
213             {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL},
214         ]
215         info_dict = _make_result(formats)
216
217         ydl = YDL({'format': 'bestvideo'})
218         ydl.process_ie_result(info_dict.copy())
219         downloaded = ydl.downloaded_info_dicts[0]
220         self.assertEqual(downloaded['format_id'], 'dash-video-high')
221
222         ydl = YDL({'format': 'worstvideo'})
223         ydl.process_ie_result(info_dict.copy())
224         downloaded = ydl.downloaded_info_dicts[0]
225         self.assertEqual(downloaded['format_id'], 'dash-video-low')
226
227         ydl = YDL({'format': 'bestvideo[format_id^=dash][format_id$=low]'})
228         ydl.process_ie_result(info_dict.copy())
229         downloaded = ydl.downloaded_info_dicts[0]
230         self.assertEqual(downloaded['format_id'], 'dash-video-low')
231
232         formats = [
233             {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
234         ]
235         info_dict = _make_result(formats)
236
237         ydl = YDL({'format': 'bestvideo[vcodec=avc1.123456]'})
238         ydl.process_ie_result(info_dict.copy())
239         downloaded = ydl.downloaded_info_dicts[0]
240         self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')
241
242     def test_youtube_format_selection(self):
243         order = [
244             '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '17', '36', '13',
245             # Apple HTTP Live Streaming
246             '96', '95', '94', '93', '92', '132', '151',
247             # 3D
248             '85', '84', '102', '83', '101', '82', '100',
249             # Dash video
250             '137', '248', '136', '247', '135', '246',
251             '245', '244', '134', '243', '133', '242', '160',
252             # Dash audio
253             '141', '172', '140', '171', '139',
254         ]
255
256         def format_info(f_id):
257             info = YoutubeIE._formats[f_id].copy()
258
259             # XXX: In real cases InfoExtractor._parse_mpd_formats() fills up 'acodec'
260             # and 'vcodec', while in tests such information is incomplete since
261             # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
262             # test_YoutubeDL.test_youtube_format_selection is broken without
263             # this fix
264             if 'acodec' in info and 'vcodec' not in info:
265                 info['vcodec'] = 'none'
266             elif 'vcodec' in info and 'acodec' not in info:
267                 info['acodec'] = 'none'
268
269             info['format_id'] = f_id
270             info['url'] = 'url:' + f_id
271             return info
272         formats_order = [format_info(f_id) for f_id in order]
273
274         info_dict = _make_result(list(formats_order), extractor='youtube')
275         ydl = YDL({'format': 'bestvideo+bestaudio'})
276         yie = YoutubeIE(ydl)
277         yie._sort_formats(info_dict['formats'])
278         ydl.process_ie_result(info_dict)
279         downloaded = ydl.downloaded_info_dicts[0]
280         self.assertEqual(downloaded['format_id'], '137+141')
281         self.assertEqual(downloaded['ext'], 'mp4')
282
283         info_dict = _make_result(list(formats_order), extractor='youtube')
284         ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
285         yie = YoutubeIE(ydl)
286         yie._sort_formats(info_dict['formats'])
287         ydl.process_ie_result(info_dict)
288         downloaded = ydl.downloaded_info_dicts[0]
289         self.assertEqual(downloaded['format_id'], '38')
290
291         info_dict = _make_result(list(formats_order), extractor='youtube')
292         ydl = YDL({'format': 'bestvideo/best,bestaudio'})
293         yie = YoutubeIE(ydl)
294         yie._sort_formats(info_dict['formats'])
295         ydl.process_ie_result(info_dict)
296         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
297         self.assertEqual(downloaded_ids, ['137', '141'])
298
299         info_dict = _make_result(list(formats_order), extractor='youtube')
300         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
301         yie = YoutubeIE(ydl)
302         yie._sort_formats(info_dict['formats'])
303         ydl.process_ie_result(info_dict)
304         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
305         self.assertEqual(downloaded_ids, ['137+141', '248+141'])
306
307         info_dict = _make_result(list(formats_order), extractor='youtube')
308         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
309         yie = YoutubeIE(ydl)
310         yie._sort_formats(info_dict['formats'])
311         ydl.process_ie_result(info_dict)
312         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
313         self.assertEqual(downloaded_ids, ['136+141', '247+141'])
314
315         info_dict = _make_result(list(formats_order), extractor='youtube')
316         ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
317         yie = YoutubeIE(ydl)
318         yie._sort_formats(info_dict['formats'])
319         ydl.process_ie_result(info_dict)
320         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
321         self.assertEqual(downloaded_ids, ['248+141'])
322
323         for f1, f2 in zip(formats_order, formats_order[1:]):
324             info_dict = _make_result([f1, f2], extractor='youtube')
325             ydl = YDL({'format': 'best/bestvideo'})
326             yie = YoutubeIE(ydl)
327             yie._sort_formats(info_dict['formats'])
328             ydl.process_ie_result(info_dict)
329             downloaded = ydl.downloaded_info_dicts[0]
330             self.assertEqual(downloaded['format_id'], f1['format_id'])
331
332             info_dict = _make_result([f2, f1], extractor='youtube')
333             ydl = YDL({'format': 'best/bestvideo'})
334             yie = YoutubeIE(ydl)
335             yie._sort_formats(info_dict['formats'])
336             ydl.process_ie_result(info_dict)
337             downloaded = ydl.downloaded_info_dicts[0]
338             self.assertEqual(downloaded['format_id'], f1['format_id'])
339
340     def test_audio_only_extractor_format_selection(self):
341         # For extractors with incomplete formats (all formats are audio-only or
342         # video-only) best and worst should fallback to corresponding best/worst
343         # video-only or audio-only formats (as per
344         # https://github.com/rg3/youtube-dl/pull/5556)
345         formats = [
346             {'format_id': 'low', 'ext': 'mp3', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
347             {'format_id': 'high', 'ext': 'mp3', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
348         ]
349         info_dict = _make_result(formats)
350
351         ydl = YDL({'format': 'best'})
352         ydl.process_ie_result(info_dict.copy())
353         downloaded = ydl.downloaded_info_dicts[0]
354         self.assertEqual(downloaded['format_id'], 'high')
355
356         ydl = YDL({'format': 'worst'})
357         ydl.process_ie_result(info_dict.copy())
358         downloaded = ydl.downloaded_info_dicts[0]
359         self.assertEqual(downloaded['format_id'], 'low')
360
361     def test_format_not_available(self):
362         formats = [
363             {'format_id': 'regular', 'ext': 'mp4', 'height': 360, 'url': TEST_URL},
364             {'format_id': 'video', 'ext': 'mp4', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
365         ]
366         info_dict = _make_result(formats)
367
368         # This must fail since complete video-audio format does not match filter
369         # and extractor does not provide incomplete only formats (i.e. only
370         # video-only or audio-only).
371         ydl = YDL({'format': 'best[height>360]'})
372         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
373
374     def test_format_selection_issue_10083(self):
375         # See https://github.com/rg3/youtube-dl/issues/10083
376         formats = [
377             {'format_id': 'regular', 'height': 360, 'url': TEST_URL},
378             {'format_id': 'video', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
379             {'format_id': 'audio', 'vcodec': 'none', 'url': TEST_URL},
380         ]
381         info_dict = _make_result(formats)
382
383         ydl = YDL({'format': 'best[height>360]/bestvideo[height>360]+bestaudio'})
384         ydl.process_ie_result(info_dict.copy())
385         self.assertEqual(ydl.downloaded_info_dicts[0]['format_id'], 'video+audio')
386
387     def test_invalid_format_specs(self):
388         def assert_syntax_error(format_spec):
389             ydl = YDL({'format': format_spec})
390             info_dict = _make_result([{'format_id': 'foo', 'url': TEST_URL}])
391             self.assertRaises(SyntaxError, ydl.process_ie_result, info_dict)
392
393         assert_syntax_error('bestvideo,,best')
394         assert_syntax_error('+bestaudio')
395         assert_syntax_error('bestvideo+')
396         assert_syntax_error('/')
397
398     def test_format_filtering(self):
399         formats = [
400             {'format_id': 'A', 'filesize': 500, 'width': 1000},
401             {'format_id': 'B', 'filesize': 1000, 'width': 500},
402             {'format_id': 'C', 'filesize': 1000, 'width': 400},
403             {'format_id': 'D', 'filesize': 2000, 'width': 600},
404             {'format_id': 'E', 'filesize': 3000},
405             {'format_id': 'F'},
406             {'format_id': 'G', 'filesize': 1000000},
407         ]
408         for f in formats:
409             f['url'] = 'http://_/'
410             f['ext'] = 'unknown'
411         info_dict = _make_result(formats)
412
413         ydl = YDL({'format': 'best[filesize<3000]'})
414         ydl.process_ie_result(info_dict)
415         downloaded = ydl.downloaded_info_dicts[0]
416         self.assertEqual(downloaded['format_id'], 'D')
417
418         ydl = YDL({'format': 'best[filesize<=3000]'})
419         ydl.process_ie_result(info_dict)
420         downloaded = ydl.downloaded_info_dicts[0]
421         self.assertEqual(downloaded['format_id'], 'E')
422
423         ydl = YDL({'format': 'best[filesize <= ? 3000]'})
424         ydl.process_ie_result(info_dict)
425         downloaded = ydl.downloaded_info_dicts[0]
426         self.assertEqual(downloaded['format_id'], 'F')
427
428         ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
429         ydl.process_ie_result(info_dict)
430         downloaded = ydl.downloaded_info_dicts[0]
431         self.assertEqual(downloaded['format_id'], 'B')
432
433         ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
434         ydl.process_ie_result(info_dict)
435         downloaded = ydl.downloaded_info_dicts[0]
436         self.assertEqual(downloaded['format_id'], 'C')
437
438         ydl = YDL({'format': '[filesize>?1]'})
439         ydl.process_ie_result(info_dict)
440         downloaded = ydl.downloaded_info_dicts[0]
441         self.assertEqual(downloaded['format_id'], 'G')
442
443         ydl = YDL({'format': '[filesize<1M]'})
444         ydl.process_ie_result(info_dict)
445         downloaded = ydl.downloaded_info_dicts[0]
446         self.assertEqual(downloaded['format_id'], 'E')
447
448         ydl = YDL({'format': '[filesize<1MiB]'})
449         ydl.process_ie_result(info_dict)
450         downloaded = ydl.downloaded_info_dicts[0]
451         self.assertEqual(downloaded['format_id'], 'G')
452
453         ydl = YDL({'format': 'all[width>=400][width<=600]'})
454         ydl.process_ie_result(info_dict)
455         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
456         self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
457
458         ydl = YDL({'format': 'best[height<40]'})
459         try:
460             ydl.process_ie_result(info_dict)
461         except ExtractorError:
462             pass
463         self.assertEqual(ydl.downloaded_info_dicts, [])
464
465     def test_default_format_spec(self):
466         ydl = YDL({'simulate': True})
467         self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
468
469         ydl = YDL({'outtmpl': '-'})
470         self.assertEqual(ydl._default_format_spec({}), 'best')
471
472         ydl = YDL({})
473         self.assertEqual(ydl._default_format_spec({}, download=False), 'bestvideo+bestaudio/best')
474         self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best')
475
476
477 class TestYoutubeDL(unittest.TestCase):
478     def test_subtitles(self):
479         def s_formats(lang, autocaption=False):
480             return [{
481                 'ext': ext,
482                 'url': 'http://localhost/video.%s.%s' % (lang, ext),
483                 '_auto': autocaption,
484             } for ext in ['vtt', 'srt', 'ass']]
485         subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es'])
486         auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es'])
487         info_dict = {
488             'id': 'test',
489             'title': 'Test',
490             'url': 'http://localhost/video.mp4',
491             'subtitles': subtitles,
492             'automatic_captions': auto_captions,
493             'extractor': 'TEST',
494         }
495
496         def get_info(params={}):
497             params.setdefault('simulate', True)
498             ydl = YDL(params)
499             ydl.report_warning = lambda *args, **kargs: None
500             return ydl.process_video_result(info_dict, download=False)
501
502         result = get_info()
503         self.assertFalse(result.get('requested_subtitles'))
504         self.assertEqual(result['subtitles'], subtitles)
505         self.assertEqual(result['automatic_captions'], auto_captions)
506
507         result = get_info({'writesubtitles': True})
508         subs = result['requested_subtitles']
509         self.assertTrue(subs)
510         self.assertEqual(set(subs.keys()), set(['en']))
511         self.assertTrue(subs['en'].get('data') is None)
512         self.assertEqual(subs['en']['ext'], 'ass')
513
514         result = get_info({'writesubtitles': True, 'subtitlesformat': 'foo/srt'})
515         subs = result['requested_subtitles']
516         self.assertEqual(subs['en']['ext'], 'srt')
517
518         result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
519         subs = result['requested_subtitles']
520         self.assertTrue(subs)
521         self.assertEqual(set(subs.keys()), set(['es', 'fr']))
522
523         result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
524         subs = result['requested_subtitles']
525         self.assertTrue(subs)
526         self.assertEqual(set(subs.keys()), set(['es', 'pt']))
527         self.assertFalse(subs['es']['_auto'])
528         self.assertTrue(subs['pt']['_auto'])
529
530         result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
531         subs = result['requested_subtitles']
532         self.assertTrue(subs)
533         self.assertEqual(set(subs.keys()), set(['es', 'pt']))
534         self.assertTrue(subs['es']['_auto'])
535         self.assertTrue(subs['pt']['_auto'])
536
537     def test_add_extra_info(self):
538         test_dict = {
539             'extractor': 'Foo',
540         }
541         extra_info = {
542             'extractor': 'Bar',
543             'playlist': 'funny videos',
544         }
545         YDL.add_extra_info(test_dict, extra_info)
546         self.assertEqual(test_dict['extractor'], 'Foo')
547         self.assertEqual(test_dict['playlist'], 'funny videos')
548
549     def test_prepare_filename(self):
550         info = {
551             'id': '1234',
552             'ext': 'mp4',
553             'width': None,
554             'height': 1080,
555             'title1': '$PATH',
556             'title2': '%PATH%',
557         }
558
559         def fname(templ):
560             ydl = YoutubeDL({'outtmpl': templ})
561             return ydl.prepare_filename(info)
562         self.assertEqual(fname('%(id)s.%(ext)s'), '1234.mp4')
563         self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
564         # Replace missing fields with 'NA'
565         self.assertEqual(fname('%(uploader_date)s-%(id)s.%(ext)s'), 'NA-1234.mp4')
566         self.assertEqual(fname('%(height)d.%(ext)s'), '1080.mp4')
567         self.assertEqual(fname('%(height)6d.%(ext)s'), '  1080.mp4')
568         self.assertEqual(fname('%(height)-6d.%(ext)s'), '1080  .mp4')
569         self.assertEqual(fname('%(height)06d.%(ext)s'), '001080.mp4')
570         self.assertEqual(fname('%(height) 06d.%(ext)s'), ' 01080.mp4')
571         self.assertEqual(fname('%(height)   06d.%(ext)s'), ' 01080.mp4')
572         self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
573         self.assertEqual(fname('%(height)0   6d.%(ext)s'), ' 01080.mp4')
574         self.assertEqual(fname('%(height)   0   6d.%(ext)s'), ' 01080.mp4')
575         self.assertEqual(fname('%%'), '%')
576         self.assertEqual(fname('%%%%'), '%%')
577         self.assertEqual(fname('%%(height)06d.%(ext)s'), '%(height)06d.mp4')
578         self.assertEqual(fname('%(width)06d.%(ext)s'), 'NA.mp4')
579         self.assertEqual(fname('%(width)06d.%%(ext)s'), 'NA.%(ext)s')
580         self.assertEqual(fname('%%(width)06d.%(ext)s'), '%(width)06d.mp4')
581         self.assertEqual(fname('Hello %(title1)s'), 'Hello $PATH')
582         self.assertEqual(fname('Hello %(title2)s'), 'Hello %PATH%')
583
584     def test_format_note(self):
585         ydl = YoutubeDL()
586         self.assertEqual(ydl._format_note({}), '')
587         assertRegexpMatches(self, ydl._format_note({
588             'vbr': 10,
589         }), r'^\s*10k$')
590         assertRegexpMatches(self, ydl._format_note({
591             'fps': 30,
592         }), r'^30fps$')
593
594     def test_postprocessors(self):
595         filename = 'post-processor-testfile.mp4'
596         audiofile = filename + '.mp3'
597
598         class SimplePP(PostProcessor):
599             def run(self, info):
600                 with open(audiofile, 'wt') as f:
601                     f.write('EXAMPLE')
602                 return [info['filepath']], info
603
604         def run_pp(params, PP):
605             with open(filename, 'wt') as f:
606                 f.write('EXAMPLE')
607             ydl = YoutubeDL(params)
608             ydl.add_post_processor(PP())
609             ydl.post_process(filename, {'filepath': filename})
610
611         run_pp({'keepvideo': True}, SimplePP)
612         self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
613         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
614         os.unlink(filename)
615         os.unlink(audiofile)
616
617         run_pp({'keepvideo': False}, SimplePP)
618         self.assertFalse(os.path.exists(filename), '%s exists' % filename)
619         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
620         os.unlink(audiofile)
621
622         class ModifierPP(PostProcessor):
623             def run(self, info):
624                 with open(info['filepath'], 'wt') as f:
625                     f.write('MODIFIED')
626                 return [], info
627
628         run_pp({'keepvideo': False}, ModifierPP)
629         self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
630         os.unlink(filename)
631
632     def test_match_filter(self):
633         class FilterYDL(YDL):
634             def __init__(self, *args, **kwargs):
635                 super(FilterYDL, self).__init__(*args, **kwargs)
636                 self.params['simulate'] = True
637
638             def process_info(self, info_dict):
639                 super(YDL, self).process_info(info_dict)
640
641             def _match_entry(self, info_dict, incomplete):
642                 res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
643                 if res is None:
644                     self.downloaded_info_dicts.append(info_dict)
645                 return res
646
647         first = {
648             'id': '1',
649             'url': TEST_URL,
650             'title': 'one',
651             'extractor': 'TEST',
652             'duration': 30,
653             'filesize': 10 * 1024,
654             'playlist_id': '42',
655             'uploader': "變態妍字幕版 太妍 тест",
656             'creator': "тест ' 123 ' тест--",
657         }
658         second = {
659             'id': '2',
660             'url': TEST_URL,
661             'title': 'two',
662             'extractor': 'TEST',
663             'duration': 10,
664             'description': 'foo',
665             'filesize': 5 * 1024,
666             'playlist_id': '43',
667             'uploader': "тест 123",
668         }
669         videos = [first, second]
670
671         def get_videos(filter_=None):
672             ydl = FilterYDL({'match_filter': filter_})
673             for v in videos:
674                 ydl.process_ie_result(v, download=True)
675             return [v['id'] for v in ydl.downloaded_info_dicts]
676
677         res = get_videos()
678         self.assertEqual(res, ['1', '2'])
679
680         def f(v):
681             if v['id'] == '1':
682                 return None
683             else:
684                 return 'Video id is not 1'
685         res = get_videos(f)
686         self.assertEqual(res, ['1'])
687
688         f = match_filter_func('duration < 30')
689         res = get_videos(f)
690         self.assertEqual(res, ['2'])
691
692         f = match_filter_func('description = foo')
693         res = get_videos(f)
694         self.assertEqual(res, ['2'])
695
696         f = match_filter_func('description =? foo')
697         res = get_videos(f)
698         self.assertEqual(res, ['1', '2'])
699
700         f = match_filter_func('filesize > 5KiB')
701         res = get_videos(f)
702         self.assertEqual(res, ['1'])
703
704         f = match_filter_func('playlist_id = 42')
705         res = get_videos(f)
706         self.assertEqual(res, ['1'])
707
708         f = match_filter_func('uploader = "變態妍字幕版 太妍 тест"')
709         res = get_videos(f)
710         self.assertEqual(res, ['1'])
711
712         f = match_filter_func('uploader != "變態妍字幕版 太妍 тест"')
713         res = get_videos(f)
714         self.assertEqual(res, ['2'])
715
716         f = match_filter_func('creator = "тест \' 123 \' тест--"')
717         res = get_videos(f)
718         self.assertEqual(res, ['1'])
719
720         f = match_filter_func("creator = 'тест \\' 123 \\' тест--'")
721         res = get_videos(f)
722         self.assertEqual(res, ['1'])
723
724         f = match_filter_func(r"creator = 'тест \' 123 \' тест--' & duration > 30")
725         res = get_videos(f)
726         self.assertEqual(res, [])
727
728     def test_playlist_items_selection(self):
729         entries = [{
730             'id': compat_str(i),
731             'title': compat_str(i),
732             'url': TEST_URL,
733         } for i in range(1, 5)]
734         playlist = {
735             '_type': 'playlist',
736             'id': 'test',
737             'entries': entries,
738             'extractor': 'test:playlist',
739             'extractor_key': 'test:playlist',
740             'webpage_url': 'http://example.com',
741         }
742
743         def get_ids(params):
744             ydl = YDL(params)
745             # make a copy because the dictionary can be modified
746             ydl.process_ie_result(playlist.copy())
747             return [int(v['id']) for v in ydl.downloaded_info_dicts]
748
749         result = get_ids({})
750         self.assertEqual(result, [1, 2, 3, 4])
751
752         result = get_ids({'playlistend': 10})
753         self.assertEqual(result, [1, 2, 3, 4])
754
755         result = get_ids({'playlistend': 2})
756         self.assertEqual(result, [1, 2])
757
758         result = get_ids({'playliststart': 10})
759         self.assertEqual(result, [])
760
761         result = get_ids({'playliststart': 2})
762         self.assertEqual(result, [2, 3, 4])
763
764         result = get_ids({'playlist_items': '2-4'})
765         self.assertEqual(result, [2, 3, 4])
766
767         result = get_ids({'playlist_items': '2,4'})
768         self.assertEqual(result, [2, 4])
769
770         result = get_ids({'playlist_items': '10'})
771         self.assertEqual(result, [])
772
773         result = get_ids({'playlist_items': '3-10'})
774         self.assertEqual(result, [3, 4])
775
776         result = get_ids({'playlist_items': '2-4,3-4,3'})
777         self.assertEqual(result, [2, 3, 4])
778
779     def test_urlopen_no_file_protocol(self):
780         # see https://github.com/rg3/youtube-dl/issues/8227
781         ydl = YDL()
782         self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
783
784     def test_do_not_override_ie_key_in_url_transparent(self):
785         ydl = YDL()
786
787         class Foo1IE(InfoExtractor):
788             _VALID_URL = r'foo1:'
789
790             def _real_extract(self, url):
791                 return {
792                     '_type': 'url_transparent',
793                     'url': 'foo2:',
794                     'ie_key': 'Foo2',
795                     'title': 'foo1 title',
796                     'id': 'foo1_id',
797                 }
798
799         class Foo2IE(InfoExtractor):
800             _VALID_URL = r'foo2:'
801
802             def _real_extract(self, url):
803                 return {
804                     '_type': 'url',
805                     'url': 'foo3:',
806                     'ie_key': 'Foo3',
807                 }
808
809         class Foo3IE(InfoExtractor):
810             _VALID_URL = r'foo3:'
811
812             def _real_extract(self, url):
813                 return _make_result([{'url': TEST_URL}], title='foo3 title')
814
815         ydl.add_info_extractor(Foo1IE(ydl))
816         ydl.add_info_extractor(Foo2IE(ydl))
817         ydl.add_info_extractor(Foo3IE(ydl))
818         ydl.extract_info('foo1:')
819         downloaded = ydl.downloaded_info_dicts[0]
820         self.assertEqual(downloaded['url'], TEST_URL)
821         self.assertEqual(downloaded['title'], 'foo1 title')
822         self.assertEqual(downloaded['id'], 'testid')
823         self.assertEqual(downloaded['extractor'], 'testex')
824         self.assertEqual(downloaded['extractor_key'], 'TestEx')
825
826
827 if __name__ == '__main__':
828     unittest.main()