[jove] Simplify, extract full description and add test for video that requires subscription
This commit is contained in:
Sergey M․
parent
d5638d974f
commit
fe556f1b0c
|
|
@ -1,64 +1,80 @@
|
||||||
# coding: utf-8
|
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import re
|
import re
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import determine_ext, ExtractorError
|
from ..utils import (
|
||||||
|
ExtractorError,
|
||||||
|
unified_strdate
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class JoveIE(InfoExtractor):
|
class JoveIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:www\.)?jove\.com/video/(?P<id>[0-9]+)'
|
_VALID_URL = r'https?://(?:www\.)?jove\.com/video/(?P<id>[0-9]+)'
|
||||||
_CHAPTERS_URL = 'http://www.jove.com/video-chapters?videoid={video_id:}'
|
_CHAPTERS_URL = 'http://www.jove.com/video-chapters?videoid={video_id:}'
|
||||||
_TEST = {
|
_TESTS = [
|
||||||
'url': 'http://www.jove.com/video/2744/electrode-positioning-montage-transcranial-direct-current',
|
{
|
||||||
'md5': '93723888d82dbd6ba8b3d7d0cd65dd2b',
|
'url': 'http://www.jove.com/video/2744/electrode-positioning-montage-transcranial-direct-current',
|
||||||
'info_dict': {
|
'md5': '93723888d82dbd6ba8b3d7d0cd65dd2b',
|
||||||
'id': '2744',
|
'info_dict': {
|
||||||
'ext': 'mp4',
|
'id': '2744',
|
||||||
'title': 'Electrode Positioning and Montage in Transcranial Direct Current Stimulation',
|
'ext': 'mp4',
|
||||||
'description': 'Transcranial direct current stimulation (tDCS) is an established technique to modulate cortical excitability1,2. It has been ...',
|
'title': 'Electrode Positioning and Montage in Transcranial Direct Current Stimulation',
|
||||||
'thumbnail': 're:^https?://.*\.png$',
|
'description': 'md5:015dd4509649c0908bc27f049e0262c6',
|
||||||
'upload_date': '20110523',
|
'thumbnail': 're:^https?://.*\.png$',
|
||||||
}
|
'upload_date': '20110523',
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'url': 'http://www.jove.com/video/51796/culturing-caenorhabditis-elegans-axenic-liquid-media-creation',
|
||||||
|
'md5': '914aeb356f416811d911996434811beb',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '51796',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Culturing Caenorhabditis elegans in Axenic Liquid Media and Creation of Transgenic Worms by Microparticle Bombardment',
|
||||||
|
'description': 'md5:35ff029261900583970c4023b70f1dc9',
|
||||||
|
'thumbnail': 're:^https?://.*\.png$',
|
||||||
|
'upload_date': '20140802',
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
webpage = self._download_webpage(url, video_id)
|
||||||
title = self._html_search_meta('citation_title', webpage, 'title')
|
|
||||||
thumbnail = self._og_search_thumbnail(webpage)
|
|
||||||
description = self._html_search_meta(
|
|
||||||
'description', webpage, 'description', fatal=False)
|
|
||||||
publish_date = self._html_search_meta(
|
|
||||||
'citation_publication_date', webpage, 'publish date', fatal=False)
|
|
||||||
if publish_date:
|
|
||||||
publish_date = datetime.strptime(publish_date,
|
|
||||||
'%Y/%m/%d').strftime('%Y%m%d')
|
|
||||||
|
|
||||||
# Not the same as video_id.
|
|
||||||
chapters_id = self._html_search_regex(
|
chapters_id = self._html_search_regex(
|
||||||
r'/video-chapters\?videoid=([0-9]+)', webpage, 'chapters id')
|
r'/video-chapters\?videoid=([0-9]+)', webpage, 'chapters id')
|
||||||
|
|
||||||
chapters_xml = self._download_xml(
|
chapters_xml = self._download_xml(
|
||||||
self._CHAPTERS_URL.format(video_id=chapters_id),
|
self._CHAPTERS_URL.format(video_id=chapters_id),
|
||||||
video_id, note='Downloading chapter XML',
|
video_id, note='Downloading chapters XML',
|
||||||
errnote='Failed to download chapter XML'
|
errnote='Failed to download chapters XML')
|
||||||
)
|
|
||||||
video_url = chapters_xml.attrib.get('video')
|
video_url = chapters_xml.attrib.get('video')
|
||||||
if not video_url:
|
if not video_url:
|
||||||
raise ExtractorError('Failed to get the video URL')
|
raise ExtractorError('Failed to get the video URL')
|
||||||
|
|
||||||
ext = determine_ext(video_url)
|
title = self._html_search_meta('citation_title', webpage, 'title')
|
||||||
|
thumbnail = self._og_search_thumbnail(webpage)
|
||||||
|
description = self._html_search_regex(
|
||||||
|
r'<div id="section_body_summary"><p class="jove_content">(.+?)</p>',
|
||||||
|
webpage, 'description', fatal=False)
|
||||||
|
publish_date = unified_strdate(self._html_search_meta(
|
||||||
|
'citation_publication_date', webpage, 'publish date', fatal=False))
|
||||||
|
comment_count = self._html_search_regex(
|
||||||
|
r'<meta name="num_comments" content="(\d+) Comments?"',
|
||||||
|
webpage, 'comment count', fatal=False)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': title,
|
'title': title,
|
||||||
'url': video_url,
|
'url': video_url,
|
||||||
'ext': ext,
|
|
||||||
'thumbnail': thumbnail,
|
'thumbnail': thumbnail,
|
||||||
'description': description,
|
'description': description,
|
||||||
'upload_date': publish_date,
|
'upload_date': publish_date,
|
||||||
|
'comment_count': comment_count,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -827,6 +827,7 @@ def unified_strdate(date_str):
|
||||||
'%b %dnd %Y %I:%M%p',
|
'%b %dnd %Y %I:%M%p',
|
||||||
'%b %dth %Y %I:%M%p',
|
'%b %dth %Y %I:%M%p',
|
||||||
'%Y-%m-%d',
|
'%Y-%m-%d',
|
||||||
|
'%Y/%m/%d',
|
||||||
'%d.%m.%Y',
|
'%d.%m.%Y',
|
||||||
'%d/%m/%Y',
|
'%d/%m/%Y',
|
||||||
'%Y/%m/%d %H:%M:%S',
|
'%Y/%m/%d %H:%M:%S',
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue