[animeondemand] Improve and modernize
[ytdl] / youtube_dl / YoutubeDL.py
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 from __future__ import absolute_import, unicode_literals
5
6 import collections
7 import contextlib
8 import copy
9 import datetime
10 import errno
11 import fileinput
12 import io
13 import itertools
14 import json
15 import locale
16 import operator
17 import os
18 import platform
19 import re
20 import shutil
21 import subprocess
22 import socket
23 import sys
24 import time
25 import tokenize
26 import traceback
27 import random
28
29 from string import ascii_letters
30
31 from .compat import (
32     compat_basestring,
33     compat_cookiejar,
34     compat_get_terminal_size,
35     compat_http_client,
36     compat_kwargs,
37     compat_numeric_types,
38     compat_os_name,
39     compat_str,
40     compat_tokenize_tokenize,
41     compat_urllib_error,
42     compat_urllib_request,
43     compat_urllib_request_DataHandler,
44 )
45 from .utils import (
46     age_restricted,
47     args_to_str,
48     ContentTooShortError,
49     date_from_str,
50     DateRange,
51     DEFAULT_OUTTMPL,
52     determine_ext,
53     determine_protocol,
54     DownloadError,
55     encode_compat_str,
56     encodeFilename,
57     error_to_compat_str,
58     expand_path,
59     ExtractorError,
60     format_bytes,
61     formatSeconds,
62     GeoRestrictedError,
63     int_or_none,
64     ISO3166Utils,
65     locked_file,
66     make_HTTPS_handler,
67     MaxDownloadsReached,
68     PagedList,
69     parse_filesize,
70     PerRequestProxyHandler,
71     platform_name,
72     PostProcessingError,
73     preferredencoding,
74     prepend_extension,
75     register_socks_protocols,
76     render_table,
77     replace_extension,
78     SameFileError,
79     sanitize_filename,
80     sanitize_path,
81     sanitize_url,
82     sanitized_Request,
83     std_headers,
84     subtitles_filename,
85     UnavailableVideoError,
86     url_basename,
87     version_tuple,
88     write_json_file,
89     write_string,
90     YoutubeDLCookieProcessor,
91     YoutubeDLHandler,
92 )
93 from .cache import Cache
94 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
95 from .downloader import get_suitable_downloader
96 from .downloader.rtmp import rtmpdump_version
97 from .postprocessor import (
98     FFmpegFixupM3u8PP,
99     FFmpegFixupM4aPP,
100     FFmpegFixupStretchedPP,
101     FFmpegMergerPP,
102     FFmpegPostProcessor,
103     get_postprocessor,
104 )
105 from .version import __version__
106
107 if compat_os_name == 'nt':
108     import ctypes
109
110
111 class YoutubeDL(object):
112     """YoutubeDL class.
113
114     YoutubeDL objects are the ones responsible of downloading the
115     actual video file and writing it to disk if the user has requested
116     it, among some other tasks. In most cases there should be one per
117     program. As, given a video URL, the downloader doesn't know how to
118     extract all the needed information, task that InfoExtractors do, it
119     has to pass the URL to one of them.
120
121     For this, YoutubeDL objects have a method that allows
122     InfoExtractors to be registered in a given order. When it is passed
123     a URL, the YoutubeDL object handles it to the first InfoExtractor it
124     finds that reports being able to handle it. The InfoExtractor extracts
125     all the information about the video or videos the URL refers to, and
126     YoutubeDL process the extracted information, possibly using a File
127     Downloader to download the video.
128
129     YoutubeDL objects accept a lot of parameters. In order not to saturate
130     the object constructor with arguments, it receives a dictionary of
131     options instead. These options are available through the params
132     attribute for the InfoExtractors to use. The YoutubeDL also
133     registers itself as the downloader in charge for the InfoExtractors
134     that are added to it, so this is a "mutual registration".
135
136     Available options:
137
138     username:          Username for authentication purposes.
139     password:          Password for authentication purposes.
140     videopassword:     Password for accessing a video.
141     ap_mso:            Adobe Pass multiple-system operator identifier.
142     ap_username:       Multiple-system operator account username.
143     ap_password:       Multiple-system operator account password.
144     usenetrc:          Use netrc for authentication instead.
145     verbose:           Print additional info to stdout.
146     quiet:             Do not print messages to stdout.
147     no_warnings:       Do not print out anything for warnings.
148     forceurl:          Force printing final URL.
149     forcetitle:        Force printing title.
150     forceid:           Force printing ID.
151     forcethumbnail:    Force printing thumbnail URL.
152     forcedescription:  Force printing description.
153     forcefilename:     Force printing final filename.
154     forceduration:     Force printing duration.
155     forcejson:         Force printing info_dict as JSON.
156     dump_single_json:  Force printing the info_dict of the whole playlist
157                        (or video) as a single JSON line.
158     simulate:          Do not download the video files.
159     format:            Video format code. See options.py for more information.
160     outtmpl:           Template for output names.
161     restrictfilenames: Do not allow "&" and spaces in file names
162     ignoreerrors:      Do not stop on download errors.
163     force_generic_extractor: Force downloader to use the generic extractor
164     nooverwrites:      Prevent overwriting files.
165     playliststart:     Playlist item to start at.
166     playlistend:       Playlist item to end at.
167     playlist_items:    Specific indices of playlist to download.
168     playlistreverse:   Download playlist items in reverse order.
169     playlistrandom:    Download playlist items in random order.
170     matchtitle:        Download only matching titles.
171     rejecttitle:       Reject downloads for matching titles.
172     logger:            Log messages to a logging.Logger instance.
173     logtostderr:       Log messages to stderr instead of stdout.
174     writedescription:  Write the video description to a .description file
175     writeinfojson:     Write the video description to a .info.json file
176     writeannotations:  Write the video annotations to a .annotations.xml file
177     writethumbnail:    Write the thumbnail image to a file
178     write_all_thumbnails:  Write all thumbnail formats to files
179     writesubtitles:    Write the video subtitles to a file
180     writeautomaticsub: Write the automatically generated subtitles to a file
181     allsubtitles:      Downloads all the subtitles of the video
182                        (requires writesubtitles or writeautomaticsub)
183     listsubtitles:     Lists all available subtitles for the video
184     subtitlesformat:   The format code for subtitles
185     subtitleslangs:    List of languages of the subtitles to download
186     keepvideo:         Keep the video file after post-processing
187     daterange:         A DateRange object, download only if the upload_date is in the range.
188     skip_download:     Skip the actual download of the video file
189     cachedir:          Location of the cache files in the filesystem.
190                        False to disable filesystem cache.
191     noplaylist:        Download single video instead of a playlist if in doubt.
192     age_limit:         An integer representing the user's age in years.
193                        Unsuitable videos for the given age are skipped.
194     min_views:         An integer representing the minimum view count the video
195                        must have in order to not be skipped.
196                        Videos without view count information are always
197                        downloaded. None for no limit.
198     max_views:         An integer representing the maximum view count.
199                        Videos that are more popular than that are not
200                        downloaded.
201                        Videos without view count information are always
202                        downloaded. None for no limit.
203     download_archive:  File name of a file where all downloads are recorded.
204                        Videos already present in the file are not downloaded
205                        again.
206     cookiefile:        File name where cookies should be read from and dumped to.
207     nocheckcertificate:Do not verify SSL certificates
208     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
209                        At the moment, this is only supported by YouTube.
210     proxy:             URL of the proxy server to use
211     geo_verification_proxy:  URL of the proxy to use for IP address verification
212                        on geo-restricted sites. (Experimental)
213     socket_timeout:    Time to wait for unresponsive hosts, in seconds
214     bidi_workaround:   Work around buggy terminals without bidirectional text
215                        support, using fridibi
216     debug_printtraffic:Print out sent and received HTTP traffic
217     include_ads:       Download ads as well
218     default_search:    Prepend this string if an input url is not valid.
219                        'auto' for elaborate guessing
220     encoding:          Use this encoding instead of the system-specified.
221     extract_flat:      Do not resolve URLs, return the immediate result.
222                        Pass in 'in_playlist' to only show this behavior for
223                        playlist items.
224     postprocessors:    A list of dictionaries, each with an entry
225                        * key:  The name of the postprocessor. See
226                                youtube_dl/postprocessor/__init__.py for a list.
227                        as well as any further keyword arguments for the
228                        postprocessor.
229     progress_hooks:    A list of functions that get called on download
230                        progress, with a dictionary with the entries
231                        * status: One of "downloading", "error", or "finished".
232                                  Check this first and ignore unknown values.
233
234                        If status is one of "downloading", or "finished", the
235                        following properties may also be present:
236                        * filename: The final filename (always present)
237                        * tmpfilename: The filename we're currently writing to
238                        * downloaded_bytes: Bytes on disk
239                        * total_bytes: Size of the whole file, None if unknown
240                        * total_bytes_estimate: Guess of the eventual file size,
241                                                None if unavailable.
242                        * elapsed: The number of seconds since download started.
243                        * eta: The estimated time in seconds, None if unknown
244                        * speed: The download speed in bytes/second, None if
245                                 unknown
246                        * fragment_index: The counter of the currently
247                                          downloaded video fragment.
248                        * fragment_count: The number of fragments (= individual
249                                          files that will be merged)
250
251                        Progress hooks are guaranteed to be called at least once
252                        (with status "finished") if the download is successful.
253     merge_output_format: Extension to use when merging formats.
254     fixup:             Automatically correct known faults of the file.
255                        One of:
256                        - "never": do nothing
257                        - "warn": only emit a warning
258                        - "detect_or_warn": check whether we can do anything
259                                            about it, warn otherwise (default)
260     source_address:    (Experimental) Client-side IP address to bind to.
261     call_home:         Boolean, true iff we are allowed to contact the
262                        youtube-dl servers for debugging.
263     sleep_interval:    Number of seconds to sleep before each download when
264                        used alone or a lower bound of a range for randomized
265                        sleep before each download (minimum possible number
266                        of seconds to sleep) when used along with
267                        max_sleep_interval.
268     max_sleep_interval:Upper bound of a range for randomized sleep before each
269                        download (maximum possible number of seconds to sleep).
270                        Must only be used along with sleep_interval.
271                        Actual sleep time will be a random float from range
272                        [sleep_interval; max_sleep_interval].
273     listformats:       Print an overview of available video formats and exit.
274     list_thumbnails:   Print a table of all thumbnails and exit.
275     match_filter:      A function that gets called with the info_dict of
276                        every video.
277                        If it returns a message, the video is ignored.
278                        If it returns None, the video is downloaded.
279                        match_filter_func in utils.py is one example for this.
280     no_color:          Do not emit color codes in output.
281     geo_bypass:        Bypass geographic restriction via faking X-Forwarded-For
282                        HTTP header (experimental)
283     geo_bypass_country:
284                        Two-letter ISO 3166-2 country code that will be used for
285                        explicit geographic restriction bypassing via faking
286                        X-Forwarded-For HTTP header (experimental)
287
288     The following options determine which downloader is picked:
289     external_downloader: Executable of the external downloader to call.
290                        None or unset for standard (built-in) downloader.
291     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
292                        if True, otherwise use ffmpeg/avconv if False, otherwise
293                        use downloader suggested by extractor if None.
294
295     The following parameters are not used by YoutubeDL itself, they are used by
296     the downloader (see youtube_dl/downloader/common.py):
297     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
298     noresizebuffer, retries, continuedl, noprogress, consoletitle,
299     xattr_set_filesize, external_downloader_args, hls_use_mpegts.
300
301     The following options are used by the post processors:
302     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
303                        otherwise prefer avconv.
304     postprocessor_args: A list of additional command-line arguments for the
305                         postprocessor.
306     """
307
308     _NUMERIC_FIELDS = set((
309         'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
310         'timestamp', 'upload_year', 'upload_month', 'upload_day',
311         'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
312         'average_rating', 'comment_count', 'age_limit',
313         'start_time', 'end_time',
314         'chapter_number', 'season_number', 'episode_number',
315         'track_number', 'disc_number', 'release_year',
316         'playlist_index',
317     ))
318
319     params = None
320     _ies = []
321     _pps = []
322     _download_retcode = None
323     _num_downloads = None
324     _screen_file = None
325
326     def __init__(self, params=None, auto_init=True):
327         """Create a FileDownloader object with the given options."""
328         if params is None:
329             params = {}
330         self._ies = []
331         self._ies_instances = {}
332         self._pps = []
333         self._progress_hooks = []
334         self._download_retcode = 0
335         self._num_downloads = 0
336         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
337         self._err_file = sys.stderr
338         self.params = {
339             # Default parameters
340             'nocheckcertificate': False,
341         }
342         self.params.update(params)
343         self.cache = Cache(self)
344
345         def check_deprecated(param, option, suggestion):
346             if self.params.get(param) is not None:
347                 self.report_warning(
348                     '%s is deprecated. Use %s instead.' % (option, suggestion))
349                 return True
350             return False
351
352         if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
353             if self.params.get('geo_verification_proxy') is None:
354                 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
355
356         check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
357         check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
358         check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
359
360         if params.get('bidi_workaround', False):
361             try:
362                 import pty
363                 master, slave = pty.openpty()
364                 width = compat_get_terminal_size().columns
365                 if width is None:
366                     width_args = []
367                 else:
368                     width_args = ['-w', str(width)]
369                 sp_kwargs = dict(
370                     stdin=subprocess.PIPE,
371                     stdout=slave,
372                     stderr=self._err_file)
373                 try:
374                     self._output_process = subprocess.Popen(
375                         ['bidiv'] + width_args, **sp_kwargs
376                     )
377                 except OSError:
378                     self._output_process = subprocess.Popen(
379                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
380                 self._output_channel = os.fdopen(master, 'rb')
381             except OSError as ose:
382                 if ose.errno == errno.ENOENT:
383                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
384                 else:
385                     raise
386
387         if (sys.platform != 'win32' and
388                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
389                 not params.get('restrictfilenames', False)):
390             # Unicode filesystem API will throw errors (#1474, #13027)
391             self.report_warning(
392                 'Assuming --restrict-filenames since file system encoding '
393                 'cannot encode all characters. '
394                 'Set the LC_ALL environment variable to fix this.')
395             self.params['restrictfilenames'] = True
396
397         if isinstance(params.get('outtmpl'), bytes):
398             self.report_warning(
399                 'Parameter outtmpl is bytes, but should be a unicode string. '
400                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
401
402         self._setup_opener()
403
404         if auto_init:
405             self.print_debug_header()
406             self.add_default_info_extractors()
407
408         for pp_def_raw in self.params.get('postprocessors', []):
409             pp_class = get_postprocessor(pp_def_raw['key'])
410             pp_def = dict(pp_def_raw)
411             del pp_def['key']
412             pp = pp_class(self, **compat_kwargs(pp_def))
413             self.add_post_processor(pp)
414
415         for ph in self.params.get('progress_hooks', []):
416             self.add_progress_hook(ph)
417
418         register_socks_protocols()
419
420     def warn_if_short_id(self, argv):
421         # short YouTube ID starting with dash?
422         idxs = [
423             i for i, a in enumerate(argv)
424             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
425         if idxs:
426             correct_argv = (
427                 ['youtube-dl'] +
428                 [a for i, a in enumerate(argv) if i not in idxs] +
429                 ['--'] + [argv[i] for i in idxs]
430             )
431             self.report_warning(
432                 'Long argument string detected. '
433                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
434                 args_to_str(correct_argv))
435
436     def add_info_extractor(self, ie):
437         """Add an InfoExtractor object to the end of the list."""
438         self._ies.append(ie)
439         if not isinstance(ie, type):
440             self._ies_instances[ie.ie_key()] = ie
441             ie.set_downloader(self)
442
443     def get_info_extractor(self, ie_key):
444         """
445         Get an instance of an IE with name ie_key, it will try to get one from
446         the _ies list, if there's no instance it will create a new one and add
447         it to the extractor list.
448         """
449         ie = self._ies_instances.get(ie_key)
450         if ie is None:
451             ie = get_info_extractor(ie_key)()
452             self.add_info_extractor(ie)
453         return ie
454
455     def add_default_info_extractors(self):
456         """
457         Add the InfoExtractors returned by gen_extractors to the end of the list
458         """
459         for ie in gen_extractor_classes():
460             self.add_info_extractor(ie)
461
462     def add_post_processor(self, pp):
463         """Add a PostProcessor object to the end of the chain."""
464         self._pps.append(pp)
465         pp.set_downloader(self)
466
467     def add_progress_hook(self, ph):
468         """Add the progress hook (currently only for the file downloader)"""
469         self._progress_hooks.append(ph)
470
471     def _bidi_workaround(self, message):
472         if not hasattr(self, '_output_channel'):
473             return message
474
475         assert hasattr(self, '_output_process')
476         assert isinstance(message, compat_str)
477         line_count = message.count('\n') + 1
478         self._output_process.stdin.write((message + '\n').encode('utf-8'))
479         self._output_process.stdin.flush()
480         res = ''.join(self._output_channel.readline().decode('utf-8')
481                       for _ in range(line_count))
482         return res[:-len('\n')]
483
484     def to_screen(self, message, skip_eol=False):
485         """Print message to stdout if not in quiet mode."""
486         return self.to_stdout(message, skip_eol, check_quiet=True)
487
488     def _write_string(self, s, out=None):
489         write_string(s, out=out, encoding=self.params.get('encoding'))
490
491     def to_stdout(self, message, skip_eol=False, check_quiet=False):
492         """Print message to stdout if not in quiet mode."""
493         if self.params.get('logger'):
494             self.params['logger'].debug(message)
495         elif not check_quiet or not self.params.get('quiet', False):
496             message = self._bidi_workaround(message)
497             terminator = ['\n', ''][skip_eol]
498             output = message + terminator
499
500             self._write_string(output, self._screen_file)
501
502     def to_stderr(self, message):
503         """Print message to stderr."""
504         assert isinstance(message, compat_str)
505         if self.params.get('logger'):
506             self.params['logger'].error(message)
507         else:
508             message = self._bidi_workaround(message)
509             output = message + '\n'
510             self._write_string(output, self._err_file)
511
512     def to_console_title(self, message):
513         if not self.params.get('consoletitle', False):
514             return
515         if compat_os_name == 'nt':
516             if ctypes.windll.kernel32.GetConsoleWindow():
517                 # c_wchar_p() might not be necessary if `message` is
518                 # already of type unicode()
519                 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
520         elif 'TERM' in os.environ:
521             self._write_string('\033]0;%s\007' % message, self._screen_file)
522
523     def save_console_title(self):
524         if not self.params.get('consoletitle', False):
525             return
526         if compat_os_name != 'nt' and 'TERM' in os.environ:
527             # Save the title on stack
528             self._write_string('\033[22;0t', self._screen_file)
529
530     def restore_console_title(self):
531         if not self.params.get('consoletitle', False):
532             return
533         if compat_os_name != 'nt' and 'TERM' in os.environ:
534             # Restore the title from stack
535             self._write_string('\033[23;0t', self._screen_file)
536
537     def __enter__(self):
538         self.save_console_title()
539         return self
540
541     def __exit__(self, *args):
542         self.restore_console_title()
543
544         if self.params.get('cookiefile') is not None:
545             self.cookiejar.save()
546
547     def trouble(self, message=None, tb=None):
548         """Determine action to take when a download problem appears.
549
550         Depending on if the downloader has been configured to ignore
551         download errors or not, this method may throw an exception or
552         not when errors are found, after printing the message.
553
554         tb, if given, is additional traceback information.
555         """
556         if message is not None:
557             self.to_stderr(message)
558         if self.params.get('verbose'):
559             if tb is None:
560                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
561                     tb = ''
562                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
563                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
564                     tb += encode_compat_str(traceback.format_exc())
565                 else:
566                     tb_data = traceback.format_list(traceback.extract_stack())
567                     tb = ''.join(tb_data)
568             self.to_stderr(tb)
569         if not self.params.get('ignoreerrors', False):
570             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
571                 exc_info = sys.exc_info()[1].exc_info
572             else:
573                 exc_info = sys.exc_info()
574             raise DownloadError(message, exc_info)
575         self._download_retcode = 1
576
577     def report_warning(self, message):
578         '''
579         Print the message to stderr, it will be prefixed with 'WARNING:'
580         If stderr is a tty file the 'WARNING:' will be colored
581         '''
582         if self.params.get('logger') is not None:
583             self.params['logger'].warning(message)
584         else:
585             if self.params.get('no_warnings'):
586                 return
587             if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
588                 _msg_header = '\033[0;33mWARNING:\033[0m'
589             else:
590                 _msg_header = 'WARNING:'
591             warning_message = '%s %s' % (_msg_header, message)
592             self.to_stderr(warning_message)
593
594     def report_error(self, message, tb=None):
595         '''
596         Do the same as trouble, but prefixes the message with 'ERROR:', colored
597         in red if stderr is a tty file.
598         '''
599         if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
600             _msg_header = '\033[0;31mERROR:\033[0m'
601         else:
602             _msg_header = 'ERROR:'
603         error_message = '%s %s' % (_msg_header, message)
604         self.trouble(error_message, tb)
605
606     def report_file_already_downloaded(self, file_name):
607         """Report file has already been fully downloaded."""
608         try:
609             self.to_screen('[download] %s has already been downloaded' % file_name)
610         except UnicodeEncodeError:
611             self.to_screen('[download] The file has already been downloaded')
612
613     def prepare_filename(self, info_dict):
614         """Generate the output filename."""
615         try:
616             template_dict = dict(info_dict)
617
618             template_dict['epoch'] = int(time.time())
619             autonumber_size = self.params.get('autonumber_size')
620             if autonumber_size is None:
621                 autonumber_size = 5
622             template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
623             if template_dict.get('resolution') is None:
624                 if template_dict.get('width') and template_dict.get('height'):
625                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
626                 elif template_dict.get('height'):
627                     template_dict['resolution'] = '%sp' % template_dict['height']
628                 elif template_dict.get('width'):
629                     template_dict['resolution'] = '%dx?' % template_dict['width']
630
631             sanitize = lambda k, v: sanitize_filename(
632                 compat_str(v),
633                 restricted=self.params.get('restrictfilenames'),
634                 is_id=(k == 'id' or k.endswith('_id')))
635             template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
636                                  for k, v in template_dict.items()
637                                  if v is not None and not isinstance(v, (list, tuple, dict)))
638             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
639
640             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
641
642             # For fields playlist_index and autonumber convert all occurrences
643             # of %(field)s to %(field)0Nd for backward compatibility
644             field_size_compat_map = {
645                 'playlist_index': len(str(template_dict['n_entries'])),
646                 'autonumber': autonumber_size,
647             }
648             FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
649             mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
650             if mobj:
651                 outtmpl = re.sub(
652                     FIELD_SIZE_COMPAT_RE,
653                     r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
654                     outtmpl)
655
656             # Missing numeric fields used together with integer presentation types
657             # in format specification will break the argument substitution since
658             # string 'NA' is returned for missing fields. We will patch output
659             # template for missing fields to meet string presentation type.
660             for numeric_field in self._NUMERIC_FIELDS:
661                 if numeric_field not in template_dict:
662                     # As of [1] format syntax is:
663                     #  %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
664                     # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
665                     FORMAT_RE = r'''(?x)
666                         (?<!%)
667                         %
668                         \({0}\)  # mapping key
669                         (?:[#0\-+ ]+)?  # conversion flags (optional)
670                         (?:\d+)?  # minimum field width (optional)
671                         (?:\.\d+)?  # precision (optional)
672                         [hlL]?  # length modifier (optional)
673                         [diouxXeEfFgGcrs%]  # conversion type
674                     '''
675                     outtmpl = re.sub(
676                         FORMAT_RE.format(numeric_field),
677                         r'%({0})s'.format(numeric_field), outtmpl)
678
679             # expand_path translates '%%' into '%' and '$$' into '$'
680             # correspondingly that is not what we want since we need to keep
681             # '%%' intact for template dict substitution step. Working around
682             # with boundary-alike separator hack.
683             sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
684             outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
685
686             # outtmpl should be expand_path'ed before template dict substitution
687             # because meta fields may contain env variables we don't want to
688             # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
689             # title "Hello $PATH", we don't want `$PATH` to be expanded.
690             filename = expand_path(outtmpl).replace(sep, '') % template_dict
691
692             # Temporary fix for #4787
693             # 'Treat' all problem characters by passing filename through preferredencoding
694             # to workaround encoding issues with subprocess on python2 @ Windows
695             if sys.version_info < (3, 0) and sys.platform == 'win32':
696                 filename = encodeFilename(filename, True).decode(preferredencoding())
697             return sanitize_path(filename)
698         except ValueError as err:
699             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
700             return None
701
702     def _match_entry(self, info_dict, incomplete):
703         """ Returns None iff the file should be downloaded """
704
705         video_title = info_dict.get('title', info_dict.get('id', 'video'))
706         if 'title' in info_dict:
707             # This can happen when we're just evaluating the playlist
708             title = info_dict['title']
709             matchtitle = self.params.get('matchtitle', False)
710             if matchtitle:
711                 if not re.search(matchtitle, title, re.IGNORECASE):
712                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
713             rejecttitle = self.params.get('rejecttitle', False)
714             if rejecttitle:
715                 if re.search(rejecttitle, title, re.IGNORECASE):
716                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
717         date = info_dict.get('upload_date')
718         if date is not None:
719             dateRange = self.params.get('daterange', DateRange())
720             if date not in dateRange:
721                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
722         view_count = info_dict.get('view_count')
723         if view_count is not None:
724             min_views = self.params.get('min_views')
725             if min_views is not None and view_count < min_views:
726                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
727             max_views = self.params.get('max_views')
728             if max_views is not None and view_count > max_views:
729                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
730         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
731             return 'Skipping "%s" because it is age restricted' % video_title
732         if self.in_download_archive(info_dict):
733             return '%s has already been recorded in archive' % video_title
734
735         if not incomplete:
736             match_filter = self.params.get('match_filter')
737             if match_filter is not None:
738                 ret = match_filter(info_dict)
739                 if ret is not None:
740                     return ret
741
742         return None
743
744     @staticmethod
745     def add_extra_info(info_dict, extra_info):
746         '''Set the keys from extra_info in info dict if they are missing'''
747         for key, value in extra_info.items():
748             info_dict.setdefault(key, value)
749
750     def extract_info(self, url, download=True, ie_key=None, extra_info={},
751                      process=True, force_generic_extractor=False):
752         '''
753         Returns a list with a dictionary for each video we find.
754         If 'download', also downloads the videos.
755         extra_info is a dict containing the extra values to add to each result
756         '''
757
758         if not ie_key and force_generic_extractor:
759             ie_key = 'Generic'
760
761         if ie_key:
762             ies = [self.get_info_extractor(ie_key)]
763         else:
764             ies = self._ies
765
766         for ie in ies:
767             if not ie.suitable(url):
768                 continue
769
770             ie = self.get_info_extractor(ie.ie_key())
771             if not ie.working():
772                 self.report_warning('The program functionality for this site has been marked as broken, '
773                                     'and will probably not work.')
774
775             try:
776                 ie_result = ie.extract(url)
777                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
778                     break
779                 if isinstance(ie_result, list):
780                     # Backwards compatibility: old IE result format
781                     ie_result = {
782                         '_type': 'compat_list',
783                         'entries': ie_result,
784                     }
785                 self.add_default_extra_info(ie_result, ie, url)
786                 if process:
787                     return self.process_ie_result(ie_result, download, extra_info)
788                 else:
789                     return ie_result
790             except GeoRestrictedError as e:
791                 msg = e.msg
792                 if e.countries:
793                     msg += '\nThis video is available in %s.' % ', '.join(
794                         map(ISO3166Utils.short2full, e.countries))
795                 msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
796                 self.report_error(msg)
797                 break
798             except ExtractorError as e:  # An error we somewhat expected
799                 self.report_error(compat_str(e), e.format_traceback())
800                 break
801             except MaxDownloadsReached:
802                 raise
803             except Exception as e:
804                 if self.params.get('ignoreerrors', False):
805                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
806                     break
807                 else:
808                     raise
809         else:
810             self.report_error('no suitable InfoExtractor for URL %s' % url)
811
812     def add_default_extra_info(self, ie_result, ie, url):
813         self.add_extra_info(ie_result, {
814             'extractor': ie.IE_NAME,
815             'webpage_url': url,
816             'webpage_url_basename': url_basename(url),
817             'extractor_key': ie.ie_key(),
818         })
819
820     def process_ie_result(self, ie_result, download=True, extra_info={}):
821         """
822         Take the result of the ie(may be modified) and resolve all unresolved
823         references (URLs, playlist items).
824
825         It will also download the videos if 'download'.
826         Returns the resolved ie_result.
827         """
828         result_type = ie_result.get('_type', 'video')
829
830         if result_type in ('url', 'url_transparent'):
831             ie_result['url'] = sanitize_url(ie_result['url'])
832             extract_flat = self.params.get('extract_flat', False)
833             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
834                     extract_flat is True):
835                 if self.params.get('forcejson', False):
836                     self.to_stdout(json.dumps(ie_result))
837                 return ie_result
838
839         if result_type == 'video':
840             self.add_extra_info(ie_result, extra_info)
841             return self.process_video_result(ie_result, download=download)
842         elif result_type == 'url':
843             # We have to add extra_info to the results because it may be
844             # contained in a playlist
845             return self.extract_info(ie_result['url'],
846                                      download,
847                                      ie_key=ie_result.get('ie_key'),
848                                      extra_info=extra_info)
849         elif result_type == 'url_transparent':
850             # Use the information from the embedding page
851             info = self.extract_info(
852                 ie_result['url'], ie_key=ie_result.get('ie_key'),
853                 extra_info=extra_info, download=False, process=False)
854
855             # extract_info may return None when ignoreerrors is enabled and
856             # extraction failed with an error, don't crash and return early
857             # in this case
858             if not info:
859                 return info
860
861             force_properties = dict(
862                 (k, v) for k, v in ie_result.items() if v is not None)
863             for f in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
864                 if f in force_properties:
865                     del force_properties[f]
866             new_result = info.copy()
867             new_result.update(force_properties)
868
869             # Extracted info may not be a video result (i.e.
870             # info.get('_type', 'video') != video) but rather an url or
871             # url_transparent. In such cases outer metadata (from ie_result)
872             # should be propagated to inner one (info). For this to happen
873             # _type of info should be overridden with url_transparent. This
874             # fixes issue from https://github.com/rg3/youtube-dl/pull/11163.
875             if new_result.get('_type') == 'url':
876                 new_result['_type'] = 'url_transparent'
877
878             return self.process_ie_result(
879                 new_result, download=download, extra_info=extra_info)
880         elif result_type in ('playlist', 'multi_video'):
881             # We process each entry in the playlist
882             playlist = ie_result.get('title') or ie_result.get('id')
883             self.to_screen('[download] Downloading playlist: %s' % playlist)
884
885             playlist_results = []
886
887             playliststart = self.params.get('playliststart', 1) - 1
888             playlistend = self.params.get('playlistend')
889             # For backwards compatibility, interpret -1 as whole list
890             if playlistend == -1:
891                 playlistend = None
892
893             playlistitems_str = self.params.get('playlist_items')
894             playlistitems = None
895             if playlistitems_str is not None:
896                 def iter_playlistitems(format):
897                     for string_segment in format.split(','):
898                         if '-' in string_segment:
899                             start, end = string_segment.split('-')
900                             for item in range(int(start), int(end) + 1):
901                                 yield int(item)
902                         else:
903                             yield int(string_segment)
904                 playlistitems = iter_playlistitems(playlistitems_str)
905
906             ie_entries = ie_result['entries']
907             if isinstance(ie_entries, list):
908                 n_all_entries = len(ie_entries)
909                 if playlistitems:
910                     entries = [
911                         ie_entries[i - 1] for i in playlistitems
912                         if -n_all_entries <= i - 1 < n_all_entries]
913                 else:
914                     entries = ie_entries[playliststart:playlistend]
915                 n_entries = len(entries)
916                 self.to_screen(
917                     '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
918                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
919             elif isinstance(ie_entries, PagedList):
920                 if playlistitems:
921                     entries = []
922                     for item in playlistitems:
923                         entries.extend(ie_entries.getslice(
924                             item - 1, item
925                         ))
926                 else:
927                     entries = ie_entries.getslice(
928                         playliststart, playlistend)
929                 n_entries = len(entries)
930                 self.to_screen(
931                     '[%s] playlist %s: Downloading %d videos' %
932                     (ie_result['extractor'], playlist, n_entries))
933             else:  # iterable
934                 if playlistitems:
935                     entry_list = list(ie_entries)
936                     entries = [entry_list[i - 1] for i in playlistitems]
937                 else:
938                     entries = list(itertools.islice(
939                         ie_entries, playliststart, playlistend))
940                 n_entries = len(entries)
941                 self.to_screen(
942                     '[%s] playlist %s: Downloading %d videos' %
943                     (ie_result['extractor'], playlist, n_entries))
944
945             if self.params.get('playlistreverse', False):
946                 entries = entries[::-1]
947
948             if self.params.get('playlistrandom', False):
949                 random.shuffle(entries)
950
951             x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
952
953             for i, entry in enumerate(entries, 1):
954                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
955                 # This __x_forwarded_for_ip thing is a bit ugly but requires
956                 # minimal changes
957                 if x_forwarded_for:
958                     entry['__x_forwarded_for_ip'] = x_forwarded_for
959                 extra = {
960                     'n_entries': n_entries,
961                     'playlist': playlist,
962                     'playlist_id': ie_result.get('id'),
963                     'playlist_title': ie_result.get('title'),
964                     'playlist_index': i + playliststart,
965                     'extractor': ie_result['extractor'],
966                     'webpage_url': ie_result['webpage_url'],
967                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
968                     'extractor_key': ie_result['extractor_key'],
969                 }
970
971                 reason = self._match_entry(entry, incomplete=True)
972                 if reason is not None:
973                     self.to_screen('[download] ' + reason)
974                     continue
975
976                 entry_result = self.process_ie_result(entry,
977                                                       download=download,
978                                                       extra_info=extra)
979                 playlist_results.append(entry_result)
980             ie_result['entries'] = playlist_results
981             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
982             return ie_result
983         elif result_type == 'compat_list':
984             self.report_warning(
985                 'Extractor %s returned a compat_list result. '
986                 'It needs to be updated.' % ie_result.get('extractor'))
987
988             def _fixup(r):
989                 self.add_extra_info(
990                     r,
991                     {
992                         'extractor': ie_result['extractor'],
993                         'webpage_url': ie_result['webpage_url'],
994                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
995                         'extractor_key': ie_result['extractor_key'],
996                     }
997                 )
998                 return r
999             ie_result['entries'] = [
1000                 self.process_ie_result(_fixup(r), download, extra_info)
1001                 for r in ie_result['entries']
1002             ]
1003             return ie_result
1004         else:
1005             raise Exception('Invalid result type: %s' % result_type)
1006
1007     def _build_format_filter(self, filter_spec):
1008         " Returns a function to filter the formats according to the filter_spec "
1009
1010         OPERATORS = {
1011             '<': operator.lt,
1012             '<=': operator.le,
1013             '>': operator.gt,
1014             '>=': operator.ge,
1015             '=': operator.eq,
1016             '!=': operator.ne,
1017         }
1018         operator_rex = re.compile(r'''(?x)\s*
1019             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
1020             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1021             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1022             $
1023             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
1024         m = operator_rex.search(filter_spec)
1025         if m:
1026             try:
1027                 comparison_value = int(m.group('value'))
1028             except ValueError:
1029                 comparison_value = parse_filesize(m.group('value'))
1030                 if comparison_value is None:
1031                     comparison_value = parse_filesize(m.group('value') + 'B')
1032                 if comparison_value is None:
1033                     raise ValueError(
1034                         'Invalid value %r in format specification %r' % (
1035                             m.group('value'), filter_spec))
1036             op = OPERATORS[m.group('op')]
1037
1038         if not m:
1039             STR_OPERATORS = {
1040                 '=': operator.eq,
1041                 '!=': operator.ne,
1042                 '^=': lambda attr, value: attr.startswith(value),
1043                 '$=': lambda attr, value: attr.endswith(value),
1044                 '*=': lambda attr, value: value in attr,
1045             }
1046             str_operator_rex = re.compile(r'''(?x)
1047                 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
1048                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
1049                 \s*(?P<value>[a-zA-Z0-9._-]+)
1050                 \s*$
1051                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
1052             m = str_operator_rex.search(filter_spec)
1053             if m:
1054                 comparison_value = m.group('value')
1055                 op = STR_OPERATORS[m.group('op')]
1056
1057         if not m:
1058             raise ValueError('Invalid filter specification %r' % filter_spec)
1059
1060         def _filter(f):
1061             actual_value = f.get(m.group('key'))
1062             if actual_value is None:
1063                 return m.group('none_inclusive')
1064             return op(actual_value, comparison_value)
1065         return _filter
1066
1067     def _default_format_spec(self, info_dict, download=True):
1068         req_format_list = []
1069
1070         def can_have_partial_formats():
1071             if self.params.get('simulate', False):
1072                 return True
1073             if not download:
1074                 return True
1075             if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
1076                 return False
1077             if info_dict.get('is_live'):
1078                 return False
1079             merger = FFmpegMergerPP(self)
1080             return merger.available and merger.can_merge()
1081         if can_have_partial_formats():
1082             req_format_list.append('bestvideo+bestaudio')
1083         req_format_list.append('best')
1084         return '/'.join(req_format_list)
1085
1086     def build_format_selector(self, format_spec):
1087         def syntax_error(note, start):
1088             message = (
1089                 'Invalid format specification: '
1090                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
1091             return SyntaxError(message)
1092
1093         PICKFIRST = 'PICKFIRST'
1094         MERGE = 'MERGE'
1095         SINGLE = 'SINGLE'
1096         GROUP = 'GROUP'
1097         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1098
1099         def _parse_filter(tokens):
1100             filter_parts = []
1101             for type, string, start, _, _ in tokens:
1102                 if type == tokenize.OP and string == ']':
1103                     return ''.join(filter_parts)
1104                 else:
1105                     filter_parts.append(string)
1106
1107         def _remove_unused_ops(tokens):
1108             # Remove operators that we don't use and join them with the surrounding strings
1109             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1110             ALLOWED_OPS = ('/', '+', ',', '(', ')')
1111             last_string, last_start, last_end, last_line = None, None, None, None
1112             for type, string, start, end, line in tokens:
1113                 if type == tokenize.OP and string == '[':
1114                     if last_string:
1115                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1116                         last_string = None
1117                     yield type, string, start, end, line
1118                     # everything inside brackets will be handled by _parse_filter
1119                     for type, string, start, end, line in tokens:
1120                         yield type, string, start, end, line
1121                         if type == tokenize.OP and string == ']':
1122                             break
1123                 elif type == tokenize.OP and string in ALLOWED_OPS:
1124                     if last_string:
1125                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1126                         last_string = None
1127                     yield type, string, start, end, line
1128                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
1129                     if not last_string:
1130                         last_string = string
1131                         last_start = start
1132                         last_end = end
1133                     else:
1134                         last_string += string
1135             if last_string:
1136                 yield tokenize.NAME, last_string, last_start, last_end, last_line
1137
1138         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
1139             selectors = []
1140             current_selector = None
1141             for type, string, start, _, _ in tokens:
1142                 # ENCODING is only defined in python 3.x
1143                 if type == getattr(tokenize, 'ENCODING', None):
1144                     continue
1145                 elif type in [tokenize.NAME, tokenize.NUMBER]:
1146                     current_selector = FormatSelector(SINGLE, string, [])
1147                 elif type == tokenize.OP:
1148                     if string == ')':
1149                         if not inside_group:
1150                             # ')' will be handled by the parentheses group
1151                             tokens.restore_last_token()
1152                         break
1153                     elif inside_merge and string in ['/', ',']:
1154                         tokens.restore_last_token()
1155                         break
1156                     elif inside_choice and string == ',':
1157                         tokens.restore_last_token()
1158                         break
1159                     elif string == ',':
1160                         if not current_selector:
1161                             raise syntax_error('"," must follow a format selector', start)
1162                         selectors.append(current_selector)
1163                         current_selector = None
1164                     elif string == '/':
1165                         if not current_selector:
1166                             raise syntax_error('"/" must follow a format selector', start)
1167                         first_choice = current_selector
1168                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1169                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1170                     elif string == '[':
1171                         if not current_selector:
1172                             current_selector = FormatSelector(SINGLE, 'best', [])
1173                         format_filter = _parse_filter(tokens)
1174                         current_selector.filters.append(format_filter)
1175                     elif string == '(':
1176                         if current_selector:
1177                             raise syntax_error('Unexpected "("', start)
1178                         group = _parse_format_selection(tokens, inside_group=True)
1179                         current_selector = FormatSelector(GROUP, group, [])
1180                     elif string == '+':
1181                         video_selector = current_selector
1182                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1183                         if not video_selector or not audio_selector:
1184                             raise syntax_error('"+" must be between two format selectors', start)
1185                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1186                     else:
1187                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1188                 elif type == tokenize.ENDMARKER:
1189                     break
1190             if current_selector:
1191                 selectors.append(current_selector)
1192             return selectors
1193
1194         def _build_selector_function(selector):
1195             if isinstance(selector, list):
1196                 fs = [_build_selector_function(s) for s in selector]
1197
1198                 def selector_function(ctx):
1199                     for f in fs:
1200                         for format in f(ctx):
1201                             yield format
1202                 return selector_function
1203             elif selector.type == GROUP:
1204                 selector_function = _build_selector_function(selector.selector)
1205             elif selector.type == PICKFIRST:
1206                 fs = [_build_selector_function(s) for s in selector.selector]
1207
1208                 def selector_function(ctx):
1209                     for f in fs:
1210                         picked_formats = list(f(ctx))
1211                         if picked_formats:
1212                             return picked_formats
1213                     return []
1214             elif selector.type == SINGLE:
1215                 format_spec = selector.selector
1216
1217                 def selector_function(ctx):
1218                     formats = list(ctx['formats'])
1219                     if not formats:
1220                         return
1221                     if format_spec == 'all':
1222                         for f in formats:
1223                             yield f
1224                     elif format_spec in ['best', 'worst', None]:
1225                         format_idx = 0 if format_spec == 'worst' else -1
1226                         audiovideo_formats = [
1227                             f for f in formats
1228                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1229                         if audiovideo_formats:
1230                             yield audiovideo_formats[format_idx]
1231                         # for extractors with incomplete formats (audio only (soundcloud)
1232                         # or video only (imgur)) we will fallback to best/worst
1233                         # {video,audio}-only format
1234                         elif ctx['incomplete_formats']:
1235                             yield formats[format_idx]
1236                     elif format_spec == 'bestaudio':
1237                         audio_formats = [
1238                             f for f in formats
1239                             if f.get('vcodec') == 'none']
1240                         if audio_formats:
1241                             yield audio_formats[-1]
1242                     elif format_spec == 'worstaudio':
1243                         audio_formats = [
1244                             f for f in formats
1245                             if f.get('vcodec') == 'none']
1246                         if audio_formats:
1247                             yield audio_formats[0]
1248                     elif format_spec == 'bestvideo':
1249                         video_formats = [
1250                             f for f in formats
1251                             if f.get('acodec') == 'none']
1252                         if video_formats:
1253                             yield video_formats[-1]
1254                     elif format_spec == 'worstvideo':
1255                         video_formats = [
1256                             f for f in formats
1257                             if f.get('acodec') == 'none']
1258                         if video_formats:
1259                             yield video_formats[0]
1260                     else:
1261                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1262                         if format_spec in extensions:
1263                             filter_f = lambda f: f['ext'] == format_spec
1264                         else:
1265                             filter_f = lambda f: f['format_id'] == format_spec
1266                         matches = list(filter(filter_f, formats))
1267                         if matches:
1268                             yield matches[-1]
1269             elif selector.type == MERGE:
1270                 def _merge(formats_info):
1271                     format_1, format_2 = [f['format_id'] for f in formats_info]
1272                     # The first format must contain the video and the
1273                     # second the audio
1274                     if formats_info[0].get('vcodec') == 'none':
1275                         self.report_error('The first format must '
1276                                           'contain the video, try using '
1277                                           '"-f %s+%s"' % (format_2, format_1))
1278                         return
1279                     # Formats must be opposite (video+audio)
1280                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1281                         self.report_error(
1282                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1283                             % (format_1, format_2))
1284                         return
1285                     output_ext = (
1286                         formats_info[0]['ext']
1287                         if self.params.get('merge_output_format') is None
1288                         else self.params['merge_output_format'])
1289                     return {
1290                         'requested_formats': formats_info,
1291                         'format': '%s+%s' % (formats_info[0].get('format'),
1292                                              formats_info[1].get('format')),
1293                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1294                                                 formats_info[1].get('format_id')),
1295                         'width': formats_info[0].get('width'),
1296                         'height': formats_info[0].get('height'),
1297                         'resolution': formats_info[0].get('resolution'),
1298                         'fps': formats_info[0].get('fps'),
1299                         'vcodec': formats_info[0].get('vcodec'),
1300                         'vbr': formats_info[0].get('vbr'),
1301                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1302                         'acodec': formats_info[1].get('acodec'),
1303                         'abr': formats_info[1].get('abr'),
1304                         'ext': output_ext,
1305                     }
1306                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1307
1308                 def selector_function(ctx):
1309                     for pair in itertools.product(
1310                             video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
1311                         yield _merge(pair)
1312
1313             filters = [self._build_format_filter(f) for f in selector.filters]
1314
1315             def final_selector(ctx):
1316                 ctx_copy = copy.deepcopy(ctx)
1317                 for _filter in filters:
1318                     ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
1319                 return selector_function(ctx_copy)
1320             return final_selector
1321
1322         stream = io.BytesIO(format_spec.encode('utf-8'))
1323         try:
1324             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1325         except tokenize.TokenError:
1326             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1327
1328         class TokenIterator(object):
1329             def __init__(self, tokens):
1330                 self.tokens = tokens
1331                 self.counter = 0
1332
1333             def __iter__(self):
1334                 return self
1335
1336             def __next__(self):
1337                 if self.counter >= len(self.tokens):
1338                     raise StopIteration()
1339                 value = self.tokens[self.counter]
1340                 self.counter += 1
1341                 return value
1342
1343             next = __next__
1344
1345             def restore_last_token(self):
1346                 self.counter -= 1
1347
1348         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1349         return _build_selector_function(parsed_selector)
1350
1351     def _calc_headers(self, info_dict):
1352         res = std_headers.copy()
1353
1354         add_headers = info_dict.get('http_headers')
1355         if add_headers:
1356             res.update(add_headers)
1357
1358         cookies = self._calc_cookies(info_dict)
1359         if cookies:
1360             res['Cookie'] = cookies
1361
1362         if 'X-Forwarded-For' not in res:
1363             x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
1364             if x_forwarded_for_ip:
1365                 res['X-Forwarded-For'] = x_forwarded_for_ip
1366
1367         return res
1368
1369     def _calc_cookies(self, info_dict):
1370         pr = sanitized_Request(info_dict['url'])
1371         self.cookiejar.add_cookie_header(pr)
1372         return pr.get_header('Cookie')
1373
1374     def process_video_result(self, info_dict, download=True):
1375         assert info_dict.get('_type', 'video') == 'video'
1376
1377         if 'id' not in info_dict:
1378             raise ExtractorError('Missing "id" field in extractor result')
1379         if 'title' not in info_dict:
1380             raise ExtractorError('Missing "title" field in extractor result')
1381
1382         def report_force_conversion(field, field_not, conversion):
1383             self.report_warning(
1384                 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1385                 % (field, field_not, conversion))
1386
1387         def sanitize_string_field(info, string_field):
1388             field = info.get(string_field)
1389             if field is None or isinstance(field, compat_str):
1390                 return
1391             report_force_conversion(string_field, 'a string', 'string')
1392             info[string_field] = compat_str(field)
1393
1394         def sanitize_numeric_fields(info):
1395             for numeric_field in self._NUMERIC_FIELDS:
1396                 field = info.get(numeric_field)
1397                 if field is None or isinstance(field, compat_numeric_types):
1398                     continue
1399                 report_force_conversion(numeric_field, 'numeric', 'int')
1400                 info[numeric_field] = int_or_none(field)
1401
1402         sanitize_string_field(info_dict, 'id')
1403         sanitize_numeric_fields(info_dict)
1404
1405         if 'playlist' not in info_dict:
1406             # It isn't part of a playlist
1407             info_dict['playlist'] = None
1408             info_dict['playlist_index'] = None
1409
1410         thumbnails = info_dict.get('thumbnails')
1411         if thumbnails is None:
1412             thumbnail = info_dict.get('thumbnail')
1413             if thumbnail:
1414                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1415         if thumbnails:
1416             thumbnails.sort(key=lambda t: (
1417                 t.get('preference') if t.get('preference') is not None else -1,
1418                 t.get('width') if t.get('width') is not None else -1,
1419                 t.get('height') if t.get('height') is not None else -1,
1420                 t.get('id') if t.get('id') is not None else '', t.get('url')))
1421             for i, t in enumerate(thumbnails):
1422                 t['url'] = sanitize_url(t['url'])
1423                 if t.get('width') and t.get('height'):
1424                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1425                 if t.get('id') is None:
1426                     t['id'] = '%d' % i
1427
1428         if self.params.get('list_thumbnails'):
1429             self.list_thumbnails(info_dict)
1430             return
1431
1432         thumbnail = info_dict.get('thumbnail')
1433         if thumbnail:
1434             info_dict['thumbnail'] = sanitize_url(thumbnail)
1435         elif thumbnails:
1436             info_dict['thumbnail'] = thumbnails[-1]['url']
1437
1438         if 'display_id' not in info_dict and 'id' in info_dict:
1439             info_dict['display_id'] = info_dict['id']
1440
1441         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1442             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1443             # see http://bugs.python.org/issue1646728)
1444             try:
1445                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1446                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1447             except (ValueError, OverflowError, OSError):
1448                 pass
1449
1450         # Auto generate title fields corresponding to the *_number fields when missing
1451         # in order to always have clean titles. This is very common for TV series.
1452         for field in ('chapter', 'season', 'episode'):
1453             if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1454                 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1455
1456         subtitles = info_dict.get('subtitles')
1457         if subtitles:
1458             for _, subtitle in subtitles.items():
1459                 for subtitle_format in subtitle:
1460                     if subtitle_format.get('url'):
1461                         subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1462                     if subtitle_format.get('ext') is None:
1463                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1464
1465         if self.params.get('listsubtitles', False):
1466             if 'automatic_captions' in info_dict:
1467                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1468             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1469             return
1470         info_dict['requested_subtitles'] = self.process_subtitles(
1471             info_dict['id'], subtitles,
1472             info_dict.get('automatic_captions'))
1473
1474         # We now pick which formats have to be downloaded
1475         if info_dict.get('formats') is None:
1476             # There's only one format available
1477             formats = [info_dict]
1478         else:
1479             formats = info_dict['formats']
1480
1481         if not formats:
1482             raise ExtractorError('No video formats found!')
1483
1484         def is_wellformed(f):
1485             url = f.get('url')
1486             if not url:
1487                 self.report_warning(
1488                     '"url" field is missing or empty - skipping format, '
1489                     'there is an error in extractor')
1490                 return False
1491             if isinstance(url, bytes):
1492                 sanitize_string_field(f, 'url')
1493             return True
1494
1495         # Filter out malformed formats for better extraction robustness
1496         formats = list(filter(is_wellformed, formats))
1497
1498         formats_dict = {}
1499
1500         # We check that all the formats have the format and format_id fields
1501         for i, format in enumerate(formats):
1502             sanitize_string_field(format, 'format_id')
1503             sanitize_numeric_fields(format)
1504             format['url'] = sanitize_url(format['url'])
1505             if not format.get('format_id'):
1506                 format['format_id'] = compat_str(i)
1507             else:
1508                 # Sanitize format_id from characters used in format selector expression
1509                 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
1510             format_id = format['format_id']
1511             if format_id not in formats_dict:
1512                 formats_dict[format_id] = []
1513             formats_dict[format_id].append(format)
1514
1515         # Make sure all formats have unique format_id
1516         for format_id, ambiguous_formats in formats_dict.items():
1517             if len(ambiguous_formats) > 1:
1518                 for i, format in enumerate(ambiguous_formats):
1519                     format['format_id'] = '%s-%d' % (format_id, i)
1520
1521         for i, format in enumerate(formats):
1522             if format.get('format') is None:
1523                 format['format'] = '{id} - {res}{note}'.format(
1524                     id=format['format_id'],
1525                     res=self.format_resolution(format),
1526                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1527                 )
1528             # Automatically determine file extension if missing
1529             if format.get('ext') is None:
1530                 format['ext'] = determine_ext(format['url']).lower()
1531             # Automatically determine protocol if missing (useful for format
1532             # selection purposes)
1533             if format.get('protocol') is None:
1534                 format['protocol'] = determine_protocol(format)
1535             # Add HTTP headers, so that external programs can use them from the
1536             # json output
1537             full_format_info = info_dict.copy()
1538             full_format_info.update(format)
1539             format['http_headers'] = self._calc_headers(full_format_info)
1540         # Remove private housekeeping stuff
1541         if '__x_forwarded_for_ip' in info_dict:
1542             del info_dict['__x_forwarded_for_ip']
1543
1544         # TODO Central sorting goes here
1545
1546         if formats[0] is not info_dict:
1547             # only set the 'formats' fields if the original info_dict list them
1548             # otherwise we end up with a circular reference, the first (and unique)
1549             # element in the 'formats' field in info_dict is info_dict itself,
1550             # which can't be exported to json
1551             info_dict['formats'] = formats
1552         if self.params.get('listformats'):
1553             self.list_formats(info_dict)
1554             return
1555
1556         req_format = self.params.get('format')
1557         if req_format is None:
1558             req_format = self._default_format_spec(info_dict, download=download)
1559             if self.params.get('verbose'):
1560                 self.to_stdout('[debug] Default format spec: %s' % req_format)
1561
1562         format_selector = self.build_format_selector(req_format)
1563
1564         # While in format selection we may need to have an access to the original
1565         # format set in order to calculate some metrics or do some processing.
1566         # For now we need to be able to guess whether original formats provided
1567         # by extractor are incomplete or not (i.e. whether extractor provides only
1568         # video-only or audio-only formats) for proper formats selection for
1569         # extractors with such incomplete formats (see
1570         # https://github.com/rg3/youtube-dl/pull/5556).
1571         # Since formats may be filtered during format selection and may not match
1572         # the original formats the results may be incorrect. Thus original formats
1573         # or pre-calculated metrics should be passed to format selection routines
1574         # as well.
1575         # We will pass a context object containing all necessary additional data
1576         # instead of just formats.
1577         # This fixes incorrect format selection issue (see
1578         # https://github.com/rg3/youtube-dl/issues/10083).
1579         incomplete_formats = (
1580             # All formats are video-only or
1581             all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
1582             # all formats are audio-only
1583             all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
1584
1585         ctx = {
1586             'formats': formats,
1587             'incomplete_formats': incomplete_formats,
1588         }
1589
1590         formats_to_download = list(format_selector(ctx))
1591         if not formats_to_download:
1592             raise ExtractorError('requested format not available',
1593                                  expected=True)
1594
1595         if download:
1596             if len(formats_to_download) > 1:
1597                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1598             for format in formats_to_download:
1599                 new_info = dict(info_dict)
1600                 new_info.update(format)
1601                 self.process_info(new_info)
1602         # We update the info dict with the best quality format (backwards compatibility)
1603         info_dict.update(formats_to_download[-1])
1604         return info_dict
1605
1606     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1607         """Select the requested subtitles and their format"""
1608         available_subs = {}
1609         if normal_subtitles and self.params.get('writesubtitles'):
1610             available_subs.update(normal_subtitles)
1611         if automatic_captions and self.params.get('writeautomaticsub'):
1612             for lang, cap_info in automatic_captions.items():
1613                 if lang not in available_subs:
1614                     available_subs[lang] = cap_info
1615
1616         if (not self.params.get('writesubtitles') and not
1617                 self.params.get('writeautomaticsub') or not
1618                 available_subs):
1619             return None
1620
1621         if self.params.get('allsubtitles', False):
1622             requested_langs = available_subs.keys()
1623         else:
1624             if self.params.get('subtitleslangs', False):
1625                 requested_langs = self.params.get('subtitleslangs')
1626             elif 'en' in available_subs:
1627                 requested_langs = ['en']
1628             else:
1629                 requested_langs = [list(available_subs.keys())[0]]
1630
1631         formats_query = self.params.get('subtitlesformat', 'best')
1632         formats_preference = formats_query.split('/') if formats_query else []
1633         subs = {}
1634         for lang in requested_langs:
1635             formats = available_subs.get(lang)
1636             if formats is None:
1637                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1638                 continue
1639             for ext in formats_preference:
1640                 if ext == 'best':
1641                     f = formats[-1]
1642                     break
1643                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1644                 if matches:
1645                     f = matches[-1]
1646                     break
1647             else:
1648                 f = formats[-1]
1649                 self.report_warning(
1650                     'No subtitle format found matching "%s" for language %s, '
1651                     'using %s' % (formats_query, lang, f['ext']))
1652             subs[lang] = f
1653         return subs
1654
1655     def process_info(self, info_dict):
1656         """Process a single resolved IE result."""
1657
1658         assert info_dict.get('_type', 'video') == 'video'
1659
1660         max_downloads = self.params.get('max_downloads')
1661         if max_downloads is not None:
1662             if self._num_downloads >= int(max_downloads):
1663                 raise MaxDownloadsReached()
1664
1665         info_dict['fulltitle'] = info_dict['title']
1666         if len(info_dict['title']) > 200:
1667             info_dict['title'] = info_dict['title'][:197] + '...'
1668
1669         if 'format' not in info_dict:
1670             info_dict['format'] = info_dict['ext']
1671
1672         reason = self._match_entry(info_dict, incomplete=False)
1673         if reason is not None:
1674             self.to_screen('[download] ' + reason)
1675             return
1676
1677         self._num_downloads += 1
1678
1679         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1680
1681         # Forced printings
1682         if self.params.get('forcetitle', False):
1683             self.to_stdout(info_dict['fulltitle'])
1684         if self.params.get('forceid', False):
1685             self.to_stdout(info_dict['id'])
1686         if self.params.get('forceurl', False):
1687             if info_dict.get('requested_formats') is not None:
1688                 for f in info_dict['requested_formats']:
1689                     self.to_stdout(f['url'] + f.get('play_path', ''))
1690             else:
1691                 # For RTMP URLs, also include the playpath
1692                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1693         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1694             self.to_stdout(info_dict['thumbnail'])
1695         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1696             self.to_stdout(info_dict['description'])
1697         if self.params.get('forcefilename', False) and filename is not None:
1698             self.to_stdout(filename)
1699         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1700             self.to_stdout(formatSeconds(info_dict['duration']))
1701         if self.params.get('forceformat', False):
1702             self.to_stdout(info_dict['format'])
1703         if self.params.get('forcejson', False):
1704             self.to_stdout(json.dumps(info_dict))
1705
1706         # Do nothing else if in simulate mode
1707         if self.params.get('simulate', False):
1708             return
1709
1710         if filename is None:
1711             return
1712
1713         def ensure_dir_exists(path):
1714             try:
1715                 dn = os.path.dirname(path)
1716                 if dn and not os.path.exists(dn):
1717                     os.makedirs(dn)
1718                 return True
1719             except (OSError, IOError) as err:
1720                 self.report_error('unable to create directory ' + error_to_compat_str(err))
1721                 return False
1722
1723         if not ensure_dir_exists(sanitize_path(encodeFilename(filename))):
1724             return
1725
1726         if self.params.get('writedescription', False):
1727             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1728             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1729                 self.to_screen('[info] Video description is already present')
1730             elif info_dict.get('description') is None:
1731                 self.report_warning('There\'s no description to write.')
1732             else:
1733                 try:
1734                     self.to_screen('[info] Writing video description to: ' + descfn)
1735                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1736                         descfile.write(info_dict['description'])
1737                 except (OSError, IOError):
1738                     self.report_error('Cannot write description file ' + descfn)
1739                     return
1740
1741         if self.params.get('writeannotations', False):
1742             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1743             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1744                 self.to_screen('[info] Video annotations are already present')
1745             else:
1746                 try:
1747                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1748                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1749                         annofile.write(info_dict['annotations'])
1750                 except (KeyError, TypeError):
1751                     self.report_warning('There are no annotations to write.')
1752                 except (OSError, IOError):
1753                     self.report_error('Cannot write annotations file: ' + annofn)
1754                     return
1755
1756         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1757                                        self.params.get('writeautomaticsub')])
1758
1759         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1760             # subtitles download errors are already managed as troubles in relevant IE
1761             # that way it will silently go on when used with unsupporting IE
1762             subtitles = info_dict['requested_subtitles']
1763             ie = self.get_info_extractor(info_dict['extractor_key'])
1764             for sub_lang, sub_info in subtitles.items():
1765                 sub_format = sub_info['ext']
1766                 if sub_info.get('data') is not None:
1767                     sub_data = sub_info['data']
1768                 else:
1769                     try:
1770                         sub_data = ie._download_webpage(
1771                             sub_info['url'], info_dict['id'], note=False)
1772                     except ExtractorError as err:
1773                         self.report_warning('Unable to download subtitle for "%s": %s' %
1774                                             (sub_lang, error_to_compat_str(err.cause)))
1775                         continue
1776                 try:
1777                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1778                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1779                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1780                     else:
1781                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1782                         # Use newline='' to prevent conversion of newline characters
1783                         # See https://github.com/rg3/youtube-dl/issues/10268
1784                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
1785                             subfile.write(sub_data)
1786                 except (OSError, IOError):
1787                     self.report_error('Cannot write subtitles file ' + sub_filename)
1788                     return
1789
1790         if self.params.get('writeinfojson', False):
1791             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1792             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1793                 self.to_screen('[info] Video description metadata is already present')
1794             else:
1795                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1796                 try:
1797                     write_json_file(self.filter_requested_info(info_dict), infofn)
1798                 except (OSError, IOError):
1799                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1800                     return
1801
1802         self._write_thumbnails(info_dict, filename)
1803
1804         if not self.params.get('skip_download', False):
1805             try:
1806                 def dl(name, info):
1807                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1808                     for ph in self._progress_hooks:
1809                         fd.add_progress_hook(ph)
1810                     if self.params.get('verbose'):
1811                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1812                     return fd.download(name, info)
1813
1814                 if info_dict.get('requested_formats') is not None:
1815                     downloaded = []
1816                     success = True
1817                     merger = FFmpegMergerPP(self)
1818                     if not merger.available:
1819                         postprocessors = []
1820                         self.report_warning('You have requested multiple '
1821                                             'formats but ffmpeg or avconv are not installed.'
1822                                             ' The formats won\'t be merged.')
1823                     else:
1824                         postprocessors = [merger]
1825
1826                     def compatible_formats(formats):
1827                         video, audio = formats
1828                         # Check extension
1829                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1830                         if video_ext and audio_ext:
1831                             COMPATIBLE_EXTS = (
1832                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),
1833                                 ('webm')
1834                             )
1835                             for exts in COMPATIBLE_EXTS:
1836                                 if video_ext in exts and audio_ext in exts:
1837                                     return True
1838                         # TODO: Check acodec/vcodec
1839                         return False
1840
1841                     filename_real_ext = os.path.splitext(filename)[1][1:]
1842                     filename_wo_ext = (
1843                         os.path.splitext(filename)[0]
1844                         if filename_real_ext == info_dict['ext']
1845                         else filename)
1846                     requested_formats = info_dict['requested_formats']
1847                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1848                         info_dict['ext'] = 'mkv'
1849                         self.report_warning(
1850                             'Requested formats are incompatible for merge and will be merged into mkv.')
1851                     # Ensure filename always has a correct extension for successful merge
1852                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1853                     if os.path.exists(encodeFilename(filename)):
1854                         self.to_screen(
1855                             '[download] %s has already been downloaded and '
1856                             'merged' % filename)
1857                     else:
1858                         for f in requested_formats:
1859                             new_info = dict(info_dict)
1860                             new_info.update(f)
1861                             fname = prepend_extension(
1862                                 self.prepare_filename(new_info),
1863                                 'f%s' % f['format_id'], new_info['ext'])
1864                             if not ensure_dir_exists(fname):
1865                                 return
1866                             downloaded.append(fname)
1867                             partial_success = dl(fname, new_info)
1868                             success = success and partial_success
1869                         info_dict['__postprocessors'] = postprocessors
1870                         info_dict['__files_to_merge'] = downloaded
1871                 else:
1872                     # Just a single file
1873                     success = dl(filename, info_dict)
1874             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1875                 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1876                 return
1877             except (OSError, IOError) as err:
1878                 raise UnavailableVideoError(err)
1879             except (ContentTooShortError, ) as err:
1880                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1881                 return
1882
1883             if success and filename != '-':
1884                 # Fixup content
1885                 fixup_policy = self.params.get('fixup')
1886                 if fixup_policy is None:
1887                     fixup_policy = 'detect_or_warn'
1888
1889                 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1890
1891                 stretched_ratio = info_dict.get('stretched_ratio')
1892                 if stretched_ratio is not None and stretched_ratio != 1:
1893                     if fixup_policy == 'warn':
1894                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1895                             info_dict['id'], stretched_ratio))
1896                     elif fixup_policy == 'detect_or_warn':
1897                         stretched_pp = FFmpegFixupStretchedPP(self)
1898                         if stretched_pp.available:
1899                             info_dict.setdefault('__postprocessors', [])
1900                             info_dict['__postprocessors'].append(stretched_pp)
1901                         else:
1902                             self.report_warning(
1903                                 '%s: Non-uniform pixel ratio (%s). %s'
1904                                 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1905                     else:
1906                         assert fixup_policy in ('ignore', 'never')
1907
1908                 if (info_dict.get('requested_formats') is None and
1909                         info_dict.get('container') == 'm4a_dash'):
1910                     if fixup_policy == 'warn':
1911                         self.report_warning(
1912                             '%s: writing DASH m4a. '
1913                             'Only some players support this container.'
1914                             % info_dict['id'])
1915                     elif fixup_policy == 'detect_or_warn':
1916                         fixup_pp = FFmpegFixupM4aPP(self)
1917                         if fixup_pp.available:
1918                             info_dict.setdefault('__postprocessors', [])
1919                             info_dict['__postprocessors'].append(fixup_pp)
1920                         else:
1921                             self.report_warning(
1922                                 '%s: writing DASH m4a. '
1923                                 'Only some players support this container. %s'
1924                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1925                     else:
1926                         assert fixup_policy in ('ignore', 'never')
1927
1928                 if (info_dict.get('protocol') == 'm3u8_native' or
1929                         info_dict.get('protocol') == 'm3u8' and
1930                         self.params.get('hls_prefer_native')):
1931                     if fixup_policy == 'warn':
1932                         self.report_warning('%s: malformed AAC bitstream detected.' % (
1933                             info_dict['id']))
1934                     elif fixup_policy == 'detect_or_warn':
1935                         fixup_pp = FFmpegFixupM3u8PP(self)
1936                         if fixup_pp.available:
1937                             info_dict.setdefault('__postprocessors', [])
1938                             info_dict['__postprocessors'].append(fixup_pp)
1939                         else:
1940                             self.report_warning(
1941                                 '%s: malformed AAC bitstream detected. %s'
1942                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1943                     else:
1944                         assert fixup_policy in ('ignore', 'never')
1945
1946                 try:
1947                     self.post_process(filename, info_dict)
1948                 except (PostProcessingError) as err:
1949                     self.report_error('postprocessing: %s' % str(err))
1950                     return
1951                 self.record_download_archive(info_dict)
1952
1953     def download(self, url_list):
1954         """Download a given list of URLs."""
1955         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1956         if (len(url_list) > 1 and
1957                 outtmpl != '-' and
1958                 '%' not in outtmpl and
1959                 self.params.get('max_downloads') != 1):
1960             raise SameFileError(outtmpl)
1961
1962         for url in url_list:
1963             try:
1964                 # It also downloads the videos
1965                 res = self.extract_info(
1966                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1967             except UnavailableVideoError:
1968                 self.report_error('unable to download video')
1969             except MaxDownloadsReached:
1970                 self.to_screen('[info] Maximum number of downloaded files reached.')
1971                 raise
1972             else:
1973                 if self.params.get('dump_single_json', False):
1974                     self.to_stdout(json.dumps(res))
1975
1976         return self._download_retcode
1977
1978     def download_with_info_file(self, info_filename):
1979         with contextlib.closing(fileinput.FileInput(
1980                 [info_filename], mode='r',
1981                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1982             # FileInput doesn't have a read method, we can't call json.load
1983             info = self.filter_requested_info(json.loads('\n'.join(f)))
1984         try:
1985             self.process_ie_result(info, download=True)
1986         except DownloadError:
1987             webpage_url = info.get('webpage_url')
1988             if webpage_url is not None:
1989                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1990                 return self.download([webpage_url])
1991             else:
1992                 raise
1993         return self._download_retcode
1994
1995     @staticmethod
1996     def filter_requested_info(info_dict):
1997         return dict(
1998             (k, v) for k, v in info_dict.items()
1999             if k not in ['requested_formats', 'requested_subtitles'])
2000
2001     def post_process(self, filename, ie_info):
2002         """Run all the postprocessors on the given file."""
2003         info = dict(ie_info)
2004         info['filepath'] = filename
2005         pps_chain = []
2006         if ie_info.get('__postprocessors') is not None:
2007             pps_chain.extend(ie_info['__postprocessors'])
2008         pps_chain.extend(self._pps)
2009         for pp in pps_chain:
2010             files_to_delete = []
2011             try:
2012                 files_to_delete, info = pp.run(info)
2013             except PostProcessingError as e:
2014                 self.report_error(e.msg)
2015             if files_to_delete and not self.params.get('keepvideo', False):
2016                 for old_filename in files_to_delete:
2017                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
2018                     try:
2019                         os.remove(encodeFilename(old_filename))
2020                     except (IOError, OSError):
2021                         self.report_warning('Unable to remove downloaded original file')
2022
2023     def _make_archive_id(self, info_dict):
2024         # Future-proof against any change in case
2025         # and backwards compatibility with prior versions
2026         extractor = info_dict.get('extractor_key')
2027         if extractor is None:
2028             if 'id' in info_dict:
2029                 extractor = info_dict.get('ie_key')  # key in a playlist
2030         if extractor is None:
2031             return None  # Incomplete video information
2032         return extractor.lower() + ' ' + info_dict['id']
2033
2034     def in_download_archive(self, info_dict):
2035         fn = self.params.get('download_archive')
2036         if fn is None:
2037             return False
2038
2039         vid_id = self._make_archive_id(info_dict)
2040         if vid_id is None:
2041             return False  # Incomplete video information
2042
2043         try:
2044             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
2045                 for line in archive_file:
2046                     if line.strip() == vid_id:
2047                         return True
2048         except IOError as ioe:
2049             if ioe.errno != errno.ENOENT:
2050                 raise
2051         return False
2052
2053     def record_download_archive(self, info_dict):
2054         fn = self.params.get('download_archive')
2055         if fn is None:
2056             return
2057         vid_id = self._make_archive_id(info_dict)
2058         assert vid_id
2059         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
2060             archive_file.write(vid_id + '\n')
2061
2062     @staticmethod
2063     def format_resolution(format, default='unknown'):
2064         if format.get('vcodec') == 'none':
2065             return 'audio only'
2066         if format.get('resolution') is not None:
2067             return format['resolution']
2068         if format.get('height') is not None:
2069             if format.get('width') is not None:
2070                 res = '%sx%s' % (format['width'], format['height'])
2071             else:
2072                 res = '%sp' % format['height']
2073         elif format.get('width') is not None:
2074             res = '%dx?' % format['width']
2075         else:
2076             res = default
2077         return res
2078
2079     def _format_note(self, fdict):
2080         res = ''
2081         if fdict.get('ext') in ['f4f', 'f4m']:
2082             res += '(unsupported) '
2083         if fdict.get('language'):
2084             if res:
2085                 res += ' '
2086             res += '[%s] ' % fdict['language']
2087         if fdict.get('format_note') is not None:
2088             res += fdict['format_note'] + ' '
2089         if fdict.get('tbr') is not None:
2090             res += '%4dk ' % fdict['tbr']
2091         if fdict.get('container') is not None:
2092             if res:
2093                 res += ', '
2094             res += '%s container' % fdict['container']
2095         if (fdict.get('vcodec') is not None and
2096                 fdict.get('vcodec') != 'none'):
2097             if res:
2098                 res += ', '
2099             res += fdict['vcodec']
2100             if fdict.get('vbr') is not None:
2101                 res += '@'
2102         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
2103             res += 'video@'
2104         if fdict.get('vbr') is not None:
2105             res += '%4dk' % fdict['vbr']
2106         if fdict.get('fps') is not None:
2107             if res:
2108                 res += ', '
2109             res += '%sfps' % fdict['fps']
2110         if fdict.get('acodec') is not None:
2111             if res:
2112                 res += ', '
2113             if fdict['acodec'] == 'none':
2114                 res += 'video only'
2115             else:
2116                 res += '%-5s' % fdict['acodec']
2117         elif fdict.get('abr') is not None:
2118             if res:
2119                 res += ', '
2120             res += 'audio'
2121         if fdict.get('abr') is not None:
2122             res += '@%3dk' % fdict['abr']
2123         if fdict.get('asr') is not None:
2124             res += ' (%5dHz)' % fdict['asr']
2125         if fdict.get('filesize') is not None:
2126             if res:
2127                 res += ', '
2128             res += format_bytes(fdict['filesize'])
2129         elif fdict.get('filesize_approx') is not None:
2130             if res:
2131                 res += ', '
2132             res += '~' + format_bytes(fdict['filesize_approx'])
2133         return res
2134
2135     def list_formats(self, info_dict):
2136         formats = info_dict.get('formats', [info_dict])
2137         table = [
2138             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
2139             for f in formats
2140             if f.get('preference') is None or f['preference'] >= -1000]
2141         if len(formats) > 1:
2142             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
2143
2144         header_line = ['format code', 'extension', 'resolution', 'note']
2145         self.to_screen(
2146             '[info] Available formats for %s:\n%s' %
2147             (info_dict['id'], render_table(header_line, table)))
2148
2149     def list_thumbnails(self, info_dict):
2150         thumbnails = info_dict.get('thumbnails')
2151         if not thumbnails:
2152             self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
2153             return
2154
2155         self.to_screen(
2156             '[info] Thumbnails for %s:' % info_dict['id'])
2157         self.to_screen(render_table(
2158             ['ID', 'width', 'height', 'URL'],
2159             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
2160
2161     def list_subtitles(self, video_id, subtitles, name='subtitles'):
2162         if not subtitles:
2163             self.to_screen('%s has no %s' % (video_id, name))
2164             return
2165         self.to_screen(
2166             'Available %s for %s:' % (name, video_id))
2167         self.to_screen(render_table(
2168             ['Language', 'formats'],
2169             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
2170                 for lang, formats in subtitles.items()]))
2171
2172     def urlopen(self, req):
2173         """ Start an HTTP download """
2174         if isinstance(req, compat_basestring):
2175             req = sanitized_Request(req)
2176         return self._opener.open(req, timeout=self._socket_timeout)
2177
2178     def print_debug_header(self):
2179         if not self.params.get('verbose'):
2180             return
2181
2182         if type('') is not compat_str:
2183             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
2184             self.report_warning(
2185                 'Your Python is broken! Update to a newer and supported version')
2186
2187         stdout_encoding = getattr(
2188             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
2189         encoding_str = (
2190             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2191                 locale.getpreferredencoding(),
2192                 sys.getfilesystemencoding(),
2193                 stdout_encoding,
2194                 self.get_encoding()))
2195         write_string(encoding_str, encoding=None)
2196
2197         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
2198         if _LAZY_LOADER:
2199             self._write_string('[debug] Lazy loading extractors enabled' + '\n')
2200         try:
2201             sp = subprocess.Popen(
2202                 ['git', 'rev-parse', '--short', 'HEAD'],
2203                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2204                 cwd=os.path.dirname(os.path.abspath(__file__)))
2205             out, err = sp.communicate()
2206             out = out.decode().strip()
2207             if re.match('[0-9a-f]+', out):
2208                 self._write_string('[debug] Git HEAD: ' + out + '\n')
2209         except Exception:
2210             try:
2211                 sys.exc_clear()
2212             except Exception:
2213                 pass
2214         self._write_string('[debug] Python version %s - %s\n' % (
2215             platform.python_version(), platform_name()))
2216
2217         exe_versions = FFmpegPostProcessor.get_versions(self)
2218         exe_versions['rtmpdump'] = rtmpdump_version()
2219         exe_str = ', '.join(
2220             '%s %s' % (exe, v)
2221             for exe, v in sorted(exe_versions.items())
2222             if v
2223         )
2224         if not exe_str:
2225             exe_str = 'none'
2226         self._write_string('[debug] exe versions: %s\n' % exe_str)
2227
2228         proxy_map = {}
2229         for handler in self._opener.handlers:
2230             if hasattr(handler, 'proxies'):
2231                 proxy_map.update(handler.proxies)
2232         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2233
2234         if self.params.get('call_home', False):
2235             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2236             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2237             latest_version = self.urlopen(
2238                 'https://yt-dl.org/latest/version').read().decode('utf-8')
2239             if version_tuple(latest_version) > version_tuple(__version__):
2240                 self.report_warning(
2241                     'You are using an outdated version (newest version: %s)! '
2242                     'See https://yt-dl.org/update if you need help updating.' %
2243                     latest_version)
2244
2245     def _setup_opener(self):
2246         timeout_val = self.params.get('socket_timeout')
2247         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2248
2249         opts_cookiefile = self.params.get('cookiefile')
2250         opts_proxy = self.params.get('proxy')
2251
2252         if opts_cookiefile is None:
2253             self.cookiejar = compat_cookiejar.CookieJar()
2254         else:
2255             opts_cookiefile = expand_path(opts_cookiefile)
2256             self.cookiejar = compat_cookiejar.MozillaCookieJar(
2257                 opts_cookiefile)
2258             if os.access(opts_cookiefile, os.R_OK):
2259                 self.cookiejar.load()
2260
2261         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2262         if opts_proxy is not None:
2263             if opts_proxy == '':
2264                 proxies = {}
2265             else:
2266                 proxies = {'http': opts_proxy, 'https': opts_proxy}
2267         else:
2268             proxies = compat_urllib_request.getproxies()
2269             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2270             if 'http' in proxies and 'https' not in proxies:
2271                 proxies['https'] = proxies['http']
2272         proxy_handler = PerRequestProxyHandler(proxies)
2273
2274         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2275         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2276         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2277         data_handler = compat_urllib_request_DataHandler()
2278
2279         # When passing our own FileHandler instance, build_opener won't add the
2280         # default FileHandler and allows us to disable the file protocol, which
2281         # can be used for malicious purposes (see
2282         # https://github.com/rg3/youtube-dl/issues/8227)
2283         file_handler = compat_urllib_request.FileHandler()
2284
2285         def file_open(*args, **kwargs):
2286             raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2287         file_handler.file_open = file_open
2288
2289         opener = compat_urllib_request.build_opener(
2290             proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2291
2292         # Delete the default user-agent header, which would otherwise apply in
2293         # cases where our custom HTTP handler doesn't come into play
2294         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2295         opener.addheaders = []
2296         self._opener = opener
2297
2298     def encode(self, s):
2299         if isinstance(s, bytes):
2300             return s  # Already encoded
2301
2302         try:
2303             return s.encode(self.get_encoding())
2304         except UnicodeEncodeError as err:
2305             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2306             raise
2307
2308     def get_encoding(self):
2309         encoding = self.params.get('encoding')
2310         if encoding is None:
2311             encoding = preferredencoding()
2312         return encoding
2313
2314     def _write_thumbnails(self, info_dict, filename):
2315         if self.params.get('writethumbnail', False):
2316             thumbnails = info_dict.get('thumbnails')
2317             if thumbnails:
2318                 thumbnails = [thumbnails[-1]]
2319         elif self.params.get('write_all_thumbnails', False):
2320             thumbnails = info_dict.get('thumbnails')
2321         else:
2322             return
2323
2324         if not thumbnails:
2325             # No thumbnails present, so return immediately
2326             return
2327
2328         for t in thumbnails:
2329             thumb_ext = determine_ext(t['url'], 'jpg')
2330             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2331             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2332             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2333
2334             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2335                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2336                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2337             else:
2338                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2339                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2340                 try:
2341                     uf = self.urlopen(t['url'])
2342                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2343                         shutil.copyfileobj(uf, thumbf)
2344                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2345                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2346                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2347                     self.report_warning('Unable to download thumbnail "%s": %s' %
2348                                         (t['url'], error_to_compat_str(err)))