From e3aaace4005e7b767d28152e60d1cf6272c398ec Mon Sep 17 00:00:00 2001 From: robin Date: Thu, 5 Feb 2015 19:55:41 +0100 Subject: [PATCH 1/6] [tweakers] Add new extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/tweakers.py | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 youtube_dl/extractor/tweakers.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 5866a7617..047f7002a 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -475,6 +475,7 @@ from .tvigle import TvigleIE from .tvp import TvpIE, TvpSeriesIE from .tvplay import TVPlayIE +from .tweakers import TweakersIE from .twentyfourvideo import TwentyFourVideoIE from .twitch import ( TwitchVideoIE, diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py new file mode 100644 index 000000000..3080f9e46 --- /dev/null +++ b/youtube_dl/extractor/tweakers.py @@ -0,0 +1,41 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor + + +class TweakersIE(InfoExtractor): + _VALID_URL = r'https?://tweakers\.net/video/(?P[0-9]+).*' + _TEST = { + 'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html', + 'md5': 'f7f7f3027166a7f32f024b4ae6571ced', + 'info_dict': { + 'id': '9926', + 'ext': 'mp4', + 'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter', + # TODO more properties, either as: + # * A value + # * MD5 checksum; start the string with md5: + # * A regular expression; start the string with re: + # * Any Python type (for example int or float) + } + } + + def _real_extract(self, url): + splitted_url = re.split('.html|/', url) + del splitted_url[-1] # To remove extra '/' at the end + video_id = splitted_url[4] + title = splitted_url[5].title() # Retrieve title for URL and capitalize + splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page + player_url = '/'.join(splitted_url) + '.html' + player_page = self._download_webpage(player_url, video_id) + + return { + 'id': video_id, + 'ext': 'mp4', + 'title': title, + 'url': re.findall('http.*mp4', player_page)[0], + 'player_url': player_url + } From 3ee6e02564f7f6c54480fd4eeb94ae08a747b9b6 Mon Sep 17 00:00:00 2001 From: robin Date: Thu, 5 Feb 2015 19:59:36 +0100 Subject: [PATCH 2/6] Edit Tweakers extractor Fixed code conventions (mainly adding two or more spaces before making an inline comment) --- youtube_dl/extractor/tweakers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py index 3080f9e46..29cd04f00 100644 --- a/youtube_dl/extractor/tweakers.py +++ b/youtube_dl/extractor/tweakers.py @@ -25,10 +25,10 @@ class TweakersIE(InfoExtractor): def _real_extract(self, url): splitted_url = re.split('.html|/', url) - del splitted_url[-1] # To remove extra '/' at the end + del splitted_url[-1] # To remove extra '/' at the end video_id = splitted_url[4] - title = splitted_url[5].title() # Retrieve title for URL and capitalize - splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page + title = splitted_url[5].title() # Retrieve title for URL and capitalize + splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page player_url = '/'.join(splitted_url) + '.html' player_page = self._download_webpage(player_url, video_id) @@ -36,6 +36,6 @@ def _real_extract(self, url): 'id': video_id, 'ext': 'mp4', 'title': title, - 'url': re.findall('http.*mp4', player_page)[0], + 'url': re.findall('http.*mp4', player_page)[0], 'player_url': player_url } From c893d70805be9b8f8a10c7d7dbe39216f8976ea5 Mon Sep 17 00:00:00 2001 From: robin Date: Thu, 5 Feb 2015 21:38:35 +0100 Subject: [PATCH 3/6] Remove player-url in tweakers.py Player-url only needed for rmftp, not for regular URLs --- youtube_dl/extractor/tweakers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py index 29cd04f00..14145e617 100644 --- a/youtube_dl/extractor/tweakers.py +++ b/youtube_dl/extractor/tweakers.py @@ -37,5 +37,4 @@ def _real_extract(self, url): 'ext': 'mp4', 'title': title, 'url': re.findall('http.*mp4', player_page)[0], - 'player_url': player_url } From bc63d56ccac8805ca8394e0b41890027e6d677d2 Mon Sep 17 00:00:00 2001 From: robin Date: Thu, 5 Feb 2015 21:40:18 +0100 Subject: [PATCH 4/6] Remove unnecessary TODO-comments --- youtube_dl/extractor/tweakers.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py index 14145e617..a4e749850 100644 --- a/youtube_dl/extractor/tweakers.py +++ b/youtube_dl/extractor/tweakers.py @@ -15,11 +15,6 @@ class TweakersIE(InfoExtractor): 'id': '9926', 'ext': 'mp4', 'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter', - # TODO more properties, either as: - # * A value - # * MD5 checksum; start the string with md5: - # * A regular expression; start the string with re: - # * Any Python type (for example int or float) } } From 67ce4f8820f94961c9b094f9c652f421c9d9f6fb Mon Sep 17 00:00:00 2001 From: robin Date: Thu, 5 Feb 2015 21:49:13 +0100 Subject: [PATCH 5/6] Use match_id method instead of splitted URL --- youtube_dl/extractor/tweakers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py index a4e749850..e332d4694 100644 --- a/youtube_dl/extractor/tweakers.py +++ b/youtube_dl/extractor/tweakers.py @@ -21,7 +21,7 @@ class TweakersIE(InfoExtractor): def _real_extract(self, url): splitted_url = re.split('.html|/', url) del splitted_url[-1] # To remove extra '/' at the end - video_id = splitted_url[4] + video_id = self._match_id(url) title = splitted_url[5].title() # Retrieve title for URL and capitalize splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page player_url = '/'.join(splitted_url) + '.html' From 9c91a8fa70e3c2f29dd23809e90b3357b6c1cdc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 6 Feb 2015 03:23:42 +0600 Subject: [PATCH 6/6] [tweakers] Switch extraction to xspf playlist, extract all formats and meta (#4881) --- youtube_dl/extractor/tweakers.py | 58 ++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py index e332d4694..c80ec15cf 100644 --- a/youtube_dl/extractor/tweakers.py +++ b/youtube_dl/extractor/tweakers.py @@ -1,35 +1,65 @@ -# coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor +from ..utils import ( + xpath_text, + xpath_with_ns, + int_or_none, + float_or_none, +) class TweakersIE(InfoExtractor): - _VALID_URL = r'https?://tweakers\.net/video/(?P[0-9]+).*' + _VALID_URL = r'https?://tweakers\.net/video/(?P\d+)' _TEST = { 'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html', - 'md5': 'f7f7f3027166a7f32f024b4ae6571ced', + 'md5': '1b5afa817403bb5baa08359dca31e6df', 'info_dict': { 'id': '9926', 'ext': 'mp4', - 'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter', + 'title': 'New Nintendo 3DS XL - Op alle fronten beter', + 'description': 'md5:f97324cc71e86e11c853f0763820e3ba', + 'thumbnail': 're:^https?://.*\.jpe?g$', + 'duration': 386, } } def _real_extract(self, url): - splitted_url = re.split('.html|/', url) - del splitted_url[-1] # To remove extra '/' at the end video_id = self._match_id(url) - title = splitted_url[5].title() # Retrieve title for URL and capitalize - splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page - player_url = '/'.join(splitted_url) + '.html' - player_page = self._download_webpage(player_url, video_id) + + playlist = self._download_xml( + 'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % video_id, + video_id) + + NS_MAP = { + 'xspf': 'http://xspf.org/ns/0/', + 's1': 'http://static.streamone.nl/player/ns/0', + } + + track = playlist.find(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)) + + title = xpath_text( + track, xpath_with_ns('./xspf:title', NS_MAP), 'title') + description = xpath_text( + track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description') + thumbnail = xpath_text( + track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail') + duration = float_or_none( + xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'), + 1000) + + formats = [{ + 'url': location.text, + 'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)), + 'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))), + 'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))), + } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))] return { 'id': video_id, - 'ext': 'mp4', 'title': title, - 'url': re.findall('http.*mp4', player_page)[0], + 'description': description, + 'thumbnail': thumbnail, + 'duration': duration, + 'formats': formats, }