Merge branch 'master' into extract_info_rewrite

README.md

@@ -18,7 +18,7 @@ which means you can modify it, redistribute it or use it however you like.
     --version                print program version and exit
     -U, --update             update this program to latest version
     -i, --ignore-errors      continue on download errors
-    -r, --rate-limit LIMIT   download rate limit (e.g. 50k or 44.6m)
+    -r, --rate-limit LIMIT   maximum download rate (e.g. 50k or 44.6m)
     -R, --retries RETRIES    number of retries (default is 10)
     --buffer-size SIZE       size of download buffer (e.g. 1024 or 16k) (default
                              is 1024)
@@ -97,10 +97,16 @@ which means you can modify it, redistribute it or use it however you like.
                              requested
     --max-quality FORMAT     highest quality format to download
     -F, --list-formats       list all available formats (currently youtube only)
-    --write-srt              write video closed captions to a .srt file
+    --write-sub              write subtitle file (currently youtube only)
+    --only-sub               downloads only the subtitles (no video)
+    --all-subs               downloads all the available subtitles of the video
                              (currently youtube only)
-    --srt-lang LANG          language of the closed captions to download
+    --list-subs              lists all available subtitles for the video
-                             (optional) use IETF language tags like 'en'
+                             (currently youtube only)
+    --sub-format LANG        subtitle format [srt/sbv] (default=srt) (currently
+                             youtube only)
+    --sub-lang LANG          language of the subtitles to download (optional)
+                             use IETF language tags like 'en'
 
 ## Authentication Options:
     -u, --username USERNAME  account username

devscripts/gh-pages/update-feed.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import datetime
+
+import textwrap
+
+import json
+
+atom_template=textwrap.dedent("""\
+								<?xml version='1.0' encoding='utf-8'?>
+								<atom:feed xmlns:atom="http://www.w3.org/2005/Atom">
+									<atom:title>youtube-dl releases</atom:title>
+									<atom:id>youtube-dl-updates-feed</atom:id>
+									<atom:updated>@TIMESTAMP@</atom:updated>
+									@ENTRIES@
+								</atom:feed>""")
+
+entry_template=textwrap.dedent("""
+								<atom:entry>
+									<atom:id>youtube-dl-@VERSION@</atom:id>
+									<atom:title>New version @VERSION@</atom:title>
+									<atom:link href="http://rg3.github.com/youtube-dl" />
+									<atom:content type="xhtml">
+										<div xmlns="http://www.w3.org/1999/xhtml">
+											Downloads available at <a href="http://youtube-dl.org/downloads/@VERSION@/">http://youtube-dl.org/downloads/@VERSION@/</a>
+										</div>
+									</atom:content>
+									<atom:author>
+										<atom:name>The youtube-dl maintainers</atom:name>
+									</atom:author>
+									<atom:updated>@TIMESTAMP@</atom:updated>
+								</atom:entry>
+								""")
+
+now = datetime.datetime.now()
+now_iso = now.isoformat()
+
+atom_template = atom_template.replace('@TIMESTAMP@',now_iso)
+
+entries=[]
+
+versions_info = json.load(open('update/versions.json'))
+versions = list(versions_info['versions'].keys())
+versions.sort()
+
+for v in versions:
+	entry = entry_template.replace('@TIMESTAMP@',v.replace('.','-'))
+	entry = entry.replace('@VERSION@',v)
+	entries.append(entry)
+
+entries_str = textwrap.indent(''.join(entries), '\t')
+atom_template = atom_template.replace('@ENTRIES@', entries_str)
+
+with open('update/releases.atom','w',encoding='utf-8') as atom_file:
+	atom_file.write(atom_template)
+
+

devscripts/release.sh

@@ -69,6 +69,7 @@ ROOT=$(pwd)
     ORIGIN_URL=$(git config --get remote.origin.url)
     cd build/gh-pages
     "$ROOT/devscripts/gh-pages/add-version.py" $version
+    "$ROOT/devscripts/gh-pages/update-feed.py"
     "$ROOT/devscripts/gh-pages/sign-versions.py" < "$ROOT/updates_key.pem"
     "$ROOT/devscripts/gh-pages/generate-download.py"
     "$ROOT/devscripts/gh-pages/update-copyright.py"

test/test_download.py

@@ -20,6 +20,8 @@ from youtube_dl.utils import *
 DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json')
 PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
 
+RETRIES = 3
+
 # General configuration (from __init__, not very elegant...)
 jar = compat_cookiejar.CookieJar()
 cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
@@ -79,9 +81,8 @@ def generator(test_case):
         params.update(test_case.get('params', {}))
 
         fd = FileDownloader(params)
-        fd.add_info_extractor(ie())
+        for ie in youtube_dl.InfoExtractors.gen_extractors():
-        for ien in test_case.get('add_ie', []):
+            fd.add_info_extractor(ie)
-            fd.add_info_extractor(getattr(youtube_dl.InfoExtractors, ien + 'IE')())
         finished_hook_called = set()
         def _hook(status):
             if status['status'] == 'finished':
@@ -94,7 +95,19 @@ def generator(test_case):
             _try_rm(tc['file'] + '.part')
             _try_rm(tc['file'] + '.info.json')
         try:
-            fd.download([test_case['url']])
+            for retry in range(1, RETRIES + 1):
+                try:
+                    fd.download([test_case['url']])
+                except (DownloadError, ExtractorError) as err:
+                    if retry == RETRIES: raise
+
+                    # Check if the exception is not a network related one
+                    if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
+                        raise
+
+                    print('Retrying: {0} failed tries\n\n##########\n\n'.format(retry))
+                else:
+                    break
 
             for tc in test_cases:
                 if not test_case.get('params', {}).get('skip_download', False):

test/tests.json

@@ -308,5 +308,25 @@
     "info_dict": {
         "title": "Vulkanausbruch in Ecuador: Der \"Feuerschlund\" ist wieder aktiv"
     }
+  },
+  {
+    "name": "LiveLeak",
+    "md5":  "0813c2430bea7a46bf13acf3406992f4",
+    "url":  "http://www.liveleak.com/view?i=757_1364311680",
+    "file":  "757_1364311680.mp4",
+    "info_dict": {
+        "title": "Most unlucky car accident",
+        "description": "extremely bad day for this guy..!",
+        "uploader": "ljfriel2"
+    }
+  },
+  {
+    "name": "WorldStarHipHop",
+    "url": "http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO",
+    "file": "wshh6a7q1ny0G34ZwuIO.mp4",
+    "md5": "9d04de741161603bf7071bbf4e883186",
+    "info_dict": {
+        "title": "Video: KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick! "
+    }
   }
 ]

youtube_dl/FileDownloader.py

@@ -231,11 +231,21 @@ class FileDownloader(object):
             self.to_stderr(message)
         if self.params.get('verbose'):
             if tb is None:
-                tb_data = traceback.format_list(traceback.extract_stack())
+                if sys.exc_info()[0]:  # if .trouble has been called from an except block
-                tb = u''.join(tb_data)
+                    tb = u''
+                    if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+                        tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
+                    tb += compat_str(traceback.format_exc())
+                else:
+                    tb_data = traceback.format_list(traceback.extract_stack())
+                    tb = u''.join(tb_data)
             self.to_stderr(tb)
         if not self.params.get('ignoreerrors', False):
-            raise DownloadError(message)
+            if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+                exc_info = sys.exc_info()[1].exc_info
+            else:
+                exc_info = sys.exc_info()
+            raise DownloadError(message, exc_info)
         self._download_retcode = 1
 
     def report_warning(self, message):
@@ -250,6 +260,18 @@ class FileDownloader(object):
         warning_message=u'%s %s' % (_msg_header,message)
         self.to_stderr(warning_message)
 
+    def report_error(self, message, tb=None):
+        '''
+        Do the same as trouble, but prefixes the message with 'ERROR:', colored
+        in red if stderr is a tty file.
+        '''
+        if sys.stderr.isatty():
+            _msg_header = u'\033[0;31mERROR:\033[0m'
+        else:
+            _msg_header = u'ERROR:'
+        error_message = u'%s %s' % (_msg_header, message)
+        self.trouble(error_message, tb)
+
     def slow_down(self, start_time, byte_counter):
         """Sleep if the download speed is over the rate limit."""
         rate_limit = self.params.get('ratelimit', None)
@@ -281,7 +303,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
-            self.trouble(u'ERROR: unable to rename file')
+            self.report_error(u'unable to rename file')
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
@@ -519,7 +541,7 @@ class FileDownloader(object):
             if dn != '' and not os.path.exists(dn): # dn is already encoded
                 os.makedirs(dn)
         except (OSError, IOError) as err:
-            self.trouble(u'ERROR: unable to create directory ' + compat_str(err))
+            self.report_error(u'unable to create directory ' + compat_str(err))
             return
 
         if self.params.get('writedescription', False):
@@ -529,7 +551,7 @@ class FileDownloader(object):
                 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
                     descfile.write(info_dict['description'])
             except (OSError, IOError):
-                self.trouble(u'ERROR: Cannot write description file ' + descfn)
+                self.report_error(u'Cannot write description file ' + descfn)
                 return
 
         if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
@@ -538,14 +560,17 @@ class FileDownloader(object):
             subtitle = info_dict['subtitles'][0]
             (sub_error, sub_lang, sub) = subtitle
             sub_format = self.params.get('subtitlesformat')
-            try:
+            if sub_error:
-                sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
+                self.report_warning("Some error while getting the subtitles")
-                self.report_writesubtitles(sub_filename)
+            else:
-                with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
+                try:
-                    subfile.write(sub)
+                    sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
-            except (OSError, IOError):
+                    self.report_writesubtitles(sub_filename)
-                self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
+                    with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
-                return
+                        subfile.write(sub)
+                except (OSError, IOError):
+                    self.report_error(u'Cannot write subtitles file ' + descfn)
+                    return
             if self.params.get('onlysubtitles', False):
                 return 
 
@@ -554,14 +579,17 @@ class FileDownloader(object):
             sub_format = self.params.get('subtitlesformat')
             for subtitle in subtitles:
                 (sub_error, sub_lang, sub) = subtitle
-                try:
+                if sub_error:
-                    sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
+                    self.report_warning("Some error while getting the subtitles")
-                    self.report_writesubtitles(sub_filename)
+                else:
-                    with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
+                    try:
-                            subfile.write(sub)
+                        sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
-                except (OSError, IOError):
+                        self.report_writesubtitles(sub_filename)
-                    self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
+                        with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
-                    return
+                                subfile.write(sub)
+                    except (OSError, IOError):
+                        self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
+                        return
             if self.params.get('onlysubtitles', False):
                 return 
 
@@ -572,7 +600,7 @@ class FileDownloader(object):
                 json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
                 write_json_file(json_info_dict, encodeFilename(infofn))
             except (OSError, IOError):
-                self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
+                self.report_error(u'Cannot write metadata to JSON file ' + infofn)
                 return
 
         if not self.params.get('skip_download', False):
@@ -584,17 +612,17 @@ class FileDownloader(object):
                 except (OSError, IOError) as err:
                     raise UnavailableVideoError()
                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                    self.trouble(u'ERROR: unable to download video data: %s' % str(err))
+                    self.report_error(u'unable to download video data: %s' % str(err))
                     return
                 except (ContentTooShortError, ) as err:
-                    self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+                    self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
                     return
 
             if success:
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
-                    self.trouble(u'ERROR: postprocessing: %s' % str(err))
+                    self.report_error(u'postprocessing: %s' % str(err))
                     return
 
     def download(self, url_list):
@@ -611,6 +639,9 @@ class FileDownloader(object):
                     self.process_info(video)
                 except UnavailableVideoError:
                     self.trouble(u'\nERROR: unable to download video')
+                except MaxDownloadsReached:
+                    self.to_screen(u'[info] Maximum number of downloaded files reached.')
+                    raise
 
         return self._download_retcode
 
@@ -645,7 +676,7 @@ class FileDownloader(object):
         try:
             subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
         except (OSError, IOError):
-            self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
+            self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
             return False
 
         # Download using rtmpdump. rtmpdump returns exit code 2 when
@@ -690,7 +721,8 @@ class FileDownloader(object):
             })
             return True
         else:
-            self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
+            self.to_stderr(u"\n")
+            self.report_error(u'rtmpdump exited with code %d' % retval)
             return False
 
     def _do_download(self, filename, info_dict):
@@ -790,7 +822,7 @@ class FileDownloader(object):
                 self.report_retry(count, retries)
 
         if count > retries:
-            self.trouble(u'ERROR: giving up after %s retries' % retries)
+            self.report_error(u'giving up after %s retries' % retries)
             return False
 
         data_len = data.info().get('Content-length', None)
@@ -826,12 +858,13 @@ class FileDownloader(object):
                     filename = self.undo_temp_name(tmpfilename)
                     self.report_destination(filename)
                 except (OSError, IOError) as err:
-                    self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
+                    self.report_error(u'unable to open for writing: %s' % str(err))
                     return False
             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:
-                self.trouble(u'\nERROR: unable to write data: %s' % str(err))
+                self.to_stderr(u"\n")
+                self.report_error(u'unable to write data: %s' % str(err))
                 return False
             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
@@ -857,7 +890,8 @@ class FileDownloader(object):
             self.slow_down(start, byte_counter - resume_len)
 
         if stream is None:
-            self.trouble(u'\nERROR: Did not get any data blocks')
+            self.to_stderr(u"\n")
+            self.report_error(u'Did not get any data blocks')
             return False
         stream.close()
         self.report_finish()

youtube_dl/utils.py

@@ -311,7 +311,7 @@ def clean_html(html):
     html = re.sub('<.*?>', '', html)
     # Replace html entities
     html = unescapeHTML(html)
-    return html
+    return html.strip()
 
 
 def sanitize_open(filename, open_mode):
@@ -329,7 +329,7 @@ def sanitize_open(filename, open_mode):
             if sys.platform == 'win32':
                 import msvcrt
                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
-            return (sys.stdout, filename)
+            return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
         stream = open(encodeFilename(filename), open_mode)
         return (stream, filename)
     except (IOError, OSError) as err:
@@ -435,6 +435,7 @@ class ExtractorError(Exception):
         """ tb, if given, is the original traceback (so that it can be printed out). """
         super(ExtractorError, self).__init__(msg)
         self.traceback = tb
+        self.exc_info = sys.exc_info()  # preserve original exception
 
     def format_traceback(self):
         if self.traceback is None:
@@ -449,7 +450,10 @@ class DownloadError(Exception):
     configured to continue on errors. They will contain the appropriate
     error message.
     """
-    pass
+    def __init__(self, msg, exc_info=None):
+        """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
+        super(DownloadError, self).__init__(msg)
+        self.exc_info = exc_info
 
 
 class SameFileError(Exception):

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2013.02.25'
+__version__ = '2013.04.03'