dbrun
/
youtube-dl
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Simplify some IEs

This commit is contained in:
Philipp Hagemeister 2013-01-01 20:52:59 +01:00
parent d830b7c297
commit 5f9551719c
1 changed files with 13 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
youtube_dl/InfoExtractors.py
View File

@ -2286,10 +2286,6 @@ class MyVideoIE(InfoExtractor):
def __init__(self, downloader=None): def __init__(self, downloader=None):
InfoExtractor.__init__(self, downloader) InfoExtractor.__init__(self, downloader)
def report_download_webpage(self, video_id):
"""Report webpage download."""
self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
def report_extraction(self, video_id): def report_extraction(self, video_id):
"""Report information extraction.""" """Report information extraction."""
self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id) self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
@ -2303,13 +2299,8 @@ class MyVideoIE(InfoExtractor):
video_id = mobj.group(1) video_id = mobj.group(1)
# Get video webpage # Get video webpage
request = compat_urllib_request.Request('http://www.myvideo.de/watch/%s' % video_id) webpage_url = 'http://www.myvideo.de/watch/%s' % video_id
try: webpage = self._download_webpage(webpage_url, video_id)
self.report_download_webpage(video_id)
webpage = compat_urllib_request.urlopen(request).read().decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
return
self.report_extraction(video_id) self.report_extraction(video_id)
mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />', mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
@ -2700,10 +2691,6 @@ class XVideosIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)' _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
IE_NAME = u'xvideos' IE_NAME = u'xvideos'
def report_webpage(self, video_id):
"""Report information extraction."""
self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
def report_extraction(self, video_id): def report_extraction(self, video_id):
"""Report information extraction.""" """Report information extraction."""
self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
@ -2715,15 +2702,7 @@ class XVideosIE(InfoExtractor):
return return
video_id = mobj.group(1) video_id = mobj.group(1)
self.report_webpage(video_id) webpage = self._download_webpage(url, video_id)
request = compat_urllib_request.Request(r'http://www.xvideos.com/video' + video_id)
try:
webpage_bytes = compat_urllib_request.urlopen(request).read()
webpage = webpage_bytes.decode('utf-8', 'replace')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return
self.report_extraction(video_id) self.report_extraction(video_id)
@ -2822,7 +2801,7 @@ class SoundcloudIE(InfoExtractor):
stream_json_bytes = compat_urllib_request.urlopen(request).read() stream_json_bytes = compat_urllib_request.urlopen(request).read()
stream_json = stream_json_bytes.decode('utf-8') stream_json = stream_json_bytes.decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) self._downloader.trouble(u'ERROR: unable to download stream definitions: %s' % compat_str(err))
return return
streams = json.loads(stream_json) streams = json.loads(stream_json)
@ -3146,10 +3125,6 @@ class MTVIE(InfoExtractor):
_VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$' _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
IE_NAME = u'mtv' IE_NAME = u'mtv'
def report_webpage(self, video_id):
"""Report information extraction."""
self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
def report_extraction(self, video_id): def report_extraction(self, video_id):
"""Report information extraction.""" """Report information extraction."""
self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
@ -3162,14 +3137,8 @@ class MTVIE(InfoExtractor):
if not mobj.group('proto'): if not mobj.group('proto'):
url = 'http://' + url url = 'http://' + url
video_id = mobj.group('videoid') video_id = mobj.group('videoid')
self.report_webpage(video_id)
request = compat_urllib_request.Request(url) webpage = self._download_webpage(url, video_id)
try:
webpage = compat_urllib_request.urlopen(request).read()
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return
mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage) mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
if mobj is None: if mobj is None:
@ -3534,9 +3503,6 @@ class NBAIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:watch\.|www\.)?nba\.com/(?:nba/)?video(/[^?]*)(\?.*)?$' _VALID_URL = r'^(?:https?://)?(?:watch\.|www\.)?nba\.com/(?:nba/)?video(/[^?]*)(\?.*)?$'
IE_NAME = u'nba' IE_NAME = u'nba'
def report_extraction(self, video_id):
self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
if mobj is None: if mobj is None:
@ -3547,14 +3513,7 @@ class NBAIE(InfoExtractor):
if video_id.endswith('/index.html'): if video_id.endswith('/index.html'):
video_id = video_id[:-len('/index.html')] video_id = video_id[:-len('/index.html')]
self.report_extraction(video_id) webpage = self._download_webpage(url, video_id)
try:
urlh = compat_urllib_request.urlopen(url)
webpage_bytes = urlh.read()
webpage = webpage_bytes.decode('utf-8', 'ignore')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
return
video_url = u'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4' video_url = u'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4'
def _findProp(rexp, default=None): def _findProp(rexp, default=None):
@ -3658,9 +3617,6 @@ class JustinTVIE(InfoExtractor):
class FunnyOrDieIE(InfoExtractor): class FunnyOrDieIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?funnyordie\.com/videos/(?P<id>[0-9a-f]+)/.*$' _VALID_URL = r'^(?:https?://)?(?:www\.)?funnyordie\.com/videos/(?P<id>[0-9a-f]+)/.*$'
def report_extraction(self, video_id):
self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
if mobj is None: if mobj is None:
@ -3668,14 +3624,7 @@ class FunnyOrDieIE(InfoExtractor):
return return
video_id = mobj.group('id') video_id = mobj.group('id')
self.report_extraction(video_id) webpage = self._download_webpage(url, video_id)
try:
urlh = compat_urllib_request.urlopen(url)
webpage_bytes = urlh.read()
webpage = webpage_bytes.decode('utf-8', 'ignore')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return
m = re.search(r'<video[^>]*>\s*<source[^>]*>\s*<source src="(?P<url>[^"]+)"', webpage, re.DOTALL) m = re.search(r'<video[^>]*>\s*<source[^>]*>\s*<source src="(?P<url>[^"]+)"', webpage, re.DOTALL)
if not m: if not m:
@ -3705,9 +3654,6 @@ class FunnyOrDieIE(InfoExtractor):
class TweetReelIE(InfoExtractor): class TweetReelIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?tweetreel\.com/[?](?P<id>[0-9a-z]+)$' _VALID_URL = r'^(?:https?://)?(?:www\.)?tweetreel\.com/[?](?P<id>[0-9a-z]+)$'
def report_extraction(self, video_id):
self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
if mobj is None: if mobj is None:
@ -3715,14 +3661,7 @@ class TweetReelIE(InfoExtractor):
return return
video_id = mobj.group('id') video_id = mobj.group('id')
self.report_extraction(video_id) webpage = self._download_webpage(url, video_id)
try:
urlh = compat_urllib_request.urlopen(url)
webpage_bytes = urlh.read()
webpage = webpage_bytes.decode('utf-8', 'ignore')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return
m = re.search(r'<div id="left" status_id="([0-9]+)">', webpage) m = re.search(r'<div id="left" status_id="([0-9]+)">', webpage)
if not m: if not m:
@ -3772,34 +3711,22 @@ class SteamIE(InfoExtractor):
"""Receives a URL and returns True if suitable for this IE.""" """Receives a URL and returns True if suitable for this IE."""
return re.match(self._VALID_URL, url, re.VERBOSE) is not None return re.match(self._VALID_URL, url, re.VERBOSE) is not None
def report_download_video_page(self, game_id):
self._downloader.to_screen(u'[%s] %s: Downloading video page' % (self.IE_NAME, game_id))
def _real_extract(self, url): def _real_extract(self, url):
m = re.match(self._VALID_URL, url, re.VERBOSE) m = re.match(self._VALID_URL, url, re.VERBOSE)
urlRE = r"'movie_(?P<videoID>\d+)': \{\s*FILENAME: \"(?P<videoURL>[\w:/\.\?=]+)\"(,\s*MOVIE_NAME: \"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\}," urlRE = r"'movie_(?P<videoID>\d+)': \{\s*FILENAME: \"(?P<videoURL>[\w:/\.\?=]+)\"(,\s*MOVIE_NAME: \"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},"
gameID = m.group('gameID') gameID = m.group('gameID')
videourl = 'http://store.steampowered.com/video/%s/' % gameID videourl = 'http://store.steampowered.com/video/%s/' % gameID
try: webpage = self._download_webpage(videourl, gameID)
self.report_download_video_page(gameID)
urlh = compat_urllib_request.urlopen(videourl)
webpage_bytes = urlh.read()
webpage = webpage_bytes.decode('utf-8', 'ignore')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return
mweb = re.finditer(urlRE, webpage) mweb = re.finditer(urlRE, webpage)
namesRE = r'<span class=\"title\">(?P<videoName>[\w:/\.\?=\+\s-]+)</span>' namesRE = r'<span class=\"title\">(?P<videoName>[\w:/\.\?=\+\s-]+)</span>'
titles = list(re.finditer(namesRE, webpage)) titles = list(re.finditer(namesRE, webpage))
videos = [] videos = []
i = 0 for vid,vtitle in zip(mweb,titles):
for vid in mweb:
video_id = vid.group('videoID') video_id = vid.group('videoID')
title = titles[i].group('videoName') title = vtitle.group('videoName')
video_url=vid.group('videoURL') video_url = vid.group('videoURL')
if not video_url: if not video_url:
self._downloader.trouble(u'ERROR: Cannot find video url for %s' % video_id) self._downloader.trouble(u'ERROR: Cannot find video url for %s' % video_id)
i += 1
info = { info = {
'id':video_id, 'id':video_id,
'url':video_url, 'url':video_url,