From f1bc56c99bac05dccb01c1b68ef778eb08fbcb71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 29 Dec 2020 02:11:48 +0700 Subject: [PATCH] [youtube:tab] Restore retry on browse requests (closes #27313, closes #27564) --- youtube_dl/extractor/youtube.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 5ef58d730..77f128285 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -16,6 +16,7 @@ from ..jsinterp import JSInterpreter from ..swfinterp import SWFInterpreter from ..compat import ( compat_chr, + compat_HTTPError, compat_parse_qs, compat_urllib_parse_unquote, compat_urllib_parse_unquote_plus, @@ -3009,10 +3010,24 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): for page_num in itertools.count(1): if not continuation: break - browse = self._download_json( - 'https://www.youtube.com/browse_ajax', None, - 'Downloading page %d' % page_num, - headers=headers, query=continuation, fatal=False) + count = 0 + retries = 3 + while count <= retries: + try: + # Downloading page may result in intermittent 5xx HTTP error + # that is usually worked around with a retry + browse = self._download_json( + 'https://www.youtube.com/browse_ajax', None, + 'Downloading page %d%s' + % (page_num, ' (retry #%d)' % count if count else ''), + headers=headers, query=continuation) + break + except ExtractorError as e: + if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503): + count += 1 + if count <= retries: + continue + raise if not browse: break response = try_get(browse, lambda x: x[1]['response'], dict)