diff --git a/youtube_dl/extractor/platzi.py b/youtube_dl/extractor/platzi.py index 23c8256b5..47328c5fe 100644 --- a/youtube_dl/extractor/platzi.py +++ b/youtube_dl/extractor/platzi.py @@ -1,16 +1,23 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .common import InfoExtractor from ..compat import ( - compat_b64decode, + compat_kwargs, compat_str, + compat_urllib_parse_urlparse, ) from ..utils import ( clean_html, + dict_get, ExtractorError, + get_element_by_class, int_or_none, + parse_iso8601, str_or_none, + strip_or_none, try_get, url_or_none, urlencode_postdata, @@ -22,6 +29,42 @@ class PlatziBaseIE(InfoExtractor): _LOGIN_URL = 'https://platzi.com/login/' _NETRC_MACHINE = 'platzi' + def _raise_extractor_error(self, video_id, reason, expected=True): + raise ExtractorError('[%s] %s: %s' % (self.IE_NAME, video_id, reason), expected=expected) + + def _download_webpage(self, url_or_request, video_id, *args, **kwargs): + # CF likes Connection: keep-alive and so disfavours Py2 + # retry on 403 may get in + kwargs['expected_status'] = 403 + # header parameters required fpor Py3 to breach site's CF fence w/o 403 + headers = kwargs.get('headers') or {} + new_hdrs = {} + if 'User-Agent' not in headers: + headers['User-Agent'] = 'Mozilla/5.0' # (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.0.0 Safari/537.36' + kwargs['headers'] = new_hdrs = headers + if new_hdrs: + kwargs = compat_kwargs(kwargs) + for _ in range(2): + x = super(PlatziBaseIE, self)._download_webpage_handle(url_or_request, video_id, *args, **kwargs) + if x is False: + return x + if x[1].getcode() != 403: + break + kwargs.pop('expected_status', None) + note = kwargs.pop('note', '') + kwargs['note'] = (note or 'Downloading webpage') + ' - retrying' + kwargs = compat_kwargs(kwargs) + path = compat_urllib_parse_urlparse(x[1].geturl()) + if path == '/': + self._raise_extractor_error(video_id, 'Redirected to home page: content expired?') + elif path == '/login': + self.raise_login_required() + else: + errs = clean_html(get_element_by_class('Errorpage-text', x[0])) + if errs: + self._raise_extractor_error(video_id, errs) + return x[0] + def _real_initialize(self): self._login() @@ -75,6 +118,26 @@ class PlatziIE(PlatziBaseIE): ''' _TESTS = [{ + 'url': 'https://platzi.com/clases/1927-intro-selenium/29383-bienvenida-al-curso', + 'md5': '0af120f1ffd18a2246f19099d52b83e2', + 'info_dict': { + 'id': '29383', + 'ext': 'mp4', + 'title': 'Por qué aprender Selenium y qué verás', + 'description': 'md5:bbe91d2760052ca4054a3149a6580436', + 'timestamp': 1627400390, + 'upload_date': '20210727', + 'creator': 'Héctor Vega', + 'series': 'Curso de Introducción a Selenium con Python', + 'duration': 11700, + 'categories': list, + }, + 'params': { + 'format': 'bestvideo', + # 'skip_download': True, + }, + 'expected_warnings': ['HTTP Error 401'] + }, { 'url': 'https://platzi.com/clases/1311-next-js/12074-creando-nuestra-primera-pagina/', 'md5': '8f56448241005b561c10f11a595b37e3', 'info_dict': { @@ -84,7 +147,7 @@ class PlatziIE(PlatziBaseIE): 'description': 'md5:4c866e45034fc76412fbf6e60ae008bc', 'duration': 420, }, - 'skip': 'Requires platzi account credentials', + 'skip': 'Content expired', }, { 'url': 'https://courses.platzi.com/classes/1367-communication-codestream/13430-background/', 'info_dict': { @@ -94,10 +157,7 @@ class PlatziIE(PlatziBaseIE): 'description': 'md5:49c83c09404b15e6e71defaf87f6b305', 'duration': 360, }, - 'skip': 'Requires platzi account credentials', - 'params': { - 'skip_download': True, - }, + 'skip': 'Content expired', }] def _real_extract(self, url): @@ -105,50 +165,60 @@ class PlatziIE(PlatziBaseIE): webpage = self._download_webpage(url, lecture_id) - data = self._parse_json( + data_preloaded_state = self._parse_json( self._search_regex( - # client_data may contain "};" so that we have to try more - # strict regex first - (r'client_data\s*=\s*({.+?})\s*;\s*\n', - r'client_data\s*=\s*({.+?})\s*;'), - webpage, 'client data'), + (r'window\s*.\s*__PRELOADED_STATE__\s*=\s*({.*?});?\s* + platzi\.com/clases| # es version + courses\.platzi\.com/classes # en version + )| + platzi\.com(?:/(?Pcursos))? )/(?P[^/?\#&]+) ''' _TESTS = [{ + 'url': 'https://platzi.com/web-angular/', + 'info_dict': { + 'id': 'web-angular', + 'title': 'Frontend con Angular', + }, + 'playlist_count': 9, + }, { + 'url': 'https://platzi.com/cursos/angular/', + 'info_dict': { + 'id': '2478', + 'title': 'Curso de Fundamentos de Angular', + }, + 'playlist_count': 21, + }, { 'url': 'https://platzi.com/clases/next-js/', 'info_dict': { 'id': '1311', 'title': 'Curso de Next.js', }, 'playlist_count': 22, + 'skip': 'Oops (updating page)', }, { 'url': 'https://courses.platzi.com/classes/communication-codestream/', 'info_dict': { @@ -175,23 +263,62 @@ class PlatziCourseIE(PlatziBaseIE): 'title': 'Codestream Course', }, 'playlist_count': 14, + 'skip': 'Content expired', }] + @classmethod + def _match_valid_url(cls, url): + return re.match(cls._VALID_URL, url) + @classmethod def suitable(cls, url): return False if PlatziIE.suitable(url) else super(PlatziCourseIE, cls).suitable(url) + def __extract_things(self, webpage, thing_id, thing_pattern): + return self.playlist_from_matches( + re.finditer(thing_pattern, webpage), + playlist_id=thing_id, + playlist_title=self._og_search_title(webpage, default=None), + getter=lambda m: urljoin('https://platzi.com', m.group('path'))) + + def _extract_classes(self, webpage, course_id): + display_id = course_id + course_id = self._search_regex( + r'''(["'])courseId\1\s*:\s*(?P\d+)''', + webpage, 'course id', group='id', fatal=False) or course_id + return self.__extract_things( + webpage, course_id, + r''']+\bhref\s*=\s*['"]?(?P/clases/\d+-%s/[^/]+)''' + % (display_id, )) + + def _extract_categories(self, webpage, cat_id): + return self.__extract_things( + webpage, cat_id, + r''']+\bhref\s*=\s*['"]?(?P/cursos/[^/]+)''') + def _real_extract(self, url): - course_name = self._match_id(url) - webpage = self._download_webpage(url, course_name) + m = self._match_valid_url(url) + classes, courses, this_id = m.group('clas', 'curs', 'id') - props = self._parse_json( - self._search_regex(r'data\s*=\s*({.+?})\s*;', webpage, 'data'), - course_name)['initialProps'] + webpage = self._download_webpage(url, this_id) + if courses: + return self._extract_classes(webpage, this_id) + + if not classes: + return self._extract_categories(webpage, this_id) + + # this branch now seems always to give "Oops" pages + course_name = this_id + + initialData = self._search_regex( + (r'window.initialData\s*=\s*({.+?})\s*;\s*\n', r'window.initialData\s*=\s*({.+?})\s*;'), + webpage, 'initialData') + props = self._parse_json(initialData, course_name, default={}) + props = try_get(props, lambda x: x['initialProps'], dict) or {} entries = [] - for chapter_num, chapter in enumerate(props['concepts'], 1): + for chapter_num, chapter in enumerate(props.get('concepts') or [], 1): if not isinstance(chapter, dict): continue materials = chapter.get('materials') @@ -221,4 +348,8 @@ class PlatziCourseIE(PlatziBaseIE): course_id = compat_str(try_get(props, lambda x: x['course']['id'])) course_title = try_get(props, lambda x: x['course']['name'], compat_str) - return self.playlist_result(entries, course_id, course_title) + result = self.playlist_result(entries, course_id, course_title) + desc = clean_html(get_element_by_class('RouteDescription-content', webpage)) + if desc: + result['description'] = desc + return result