From 297f60e62c4b799bc58257813ffce452a60a250a Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 12 Aug 2019 17:50:03 +0200 Subject: [PATCH 1/9] [DeezerPlaylist] Update to new website --- youtube_dl/extractor/deezer.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index a38b2683d..458b89b75 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -18,11 +18,10 @@ class DeezerPlaylistIE(InfoExtractor): 'info_dict': { 'id': '176747451', 'title': 'Best!', - 'uploader': 'Anonymous', - 'thumbnail': r're:^https?://cdn-images\.deezer\.com/images/cover/.*\.jpg$', + 'uploader': 'anonymous', + 'thumbnail': r're:^https?://e-cdns-images\.dzcdn\.net/images/cover/.*\.jpg$', }, - 'playlist_count': 30, - 'skip': 'Only available in .de', + 'playlist_count': 29, } def _real_extract(self, url): @@ -46,25 +45,17 @@ def _real_extract(self, url): webpage, 'data JSON') data = json.loads(data_json) - playlist_title = data.get('DATA', {}).get('TITLE') - playlist_uploader = data.get('DATA', {}).get('PARENT_USERNAME') + playlist_title = data['DATA']['TITLE'] + playlist_uploader = data['DATA']['PARENT_USERNAME'] playlist_thumbnail = self._search_regex( r' Date: Tue, 13 Aug 2019 12:46:06 +0200 Subject: [PATCH 2/9] [DeezerPlaylist] Coding convention --- youtube_dl/extractor/deezer.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index 458b89b75..3a7e7d5b0 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -45,29 +45,29 @@ def _real_extract(self, url): webpage, 'data JSON') data = json.loads(data_json) - playlist_title = data['DATA']['TITLE'] - playlist_uploader = data['DATA']['PARENT_USERNAME'] + playlist_title = data.get('DATA').get('TITLE') + playlist_uploader = data.get('DATA').get('PARENT_USERNAME') playlist_thumbnail = self._search_regex( r' Date: Sun, 18 Aug 2019 23:35:39 +0200 Subject: [PATCH 3/9] Support language specific sites --- youtube_dl/extractor/deezer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index 3a7e7d5b0..0aff4074e 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -10,16 +10,15 @@ orderedSet, ) - class DeezerPlaylistIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?deezer\.com/playlist/(?P[0-9]+)' + _VALID_URL = r'https?://(?:www\.)?deezer\.com/(../)?playlist/(?P[0-9]+)' _TEST = { 'url': 'http://www.deezer.com/playlist/176747451', 'info_dict': { 'id': '176747451', 'title': 'Best!', 'uploader': 'anonymous', - 'thumbnail': r're:^https?://e-cdns-images\.dzcdn\.net/images/cover/.*\.jpg$', + 'thumbnail': r're:^https?://(e-)?cdns-images\.dzcdn\.net/images/cover/.*\.jpg$', }, 'playlist_count': 29, } From 480f2d89f6b461123e24a1288fddc287c3838d37 Mon Sep 17 00:00:00 2001 From: Lucas Date: Sun, 18 Aug 2019 23:39:06 +0200 Subject: [PATCH 4/9] Add base info extractor class --- youtube_dl/extractor/deezer.py | 46 +++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index 0aff4074e..6a21f2ffa 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -10,7 +10,31 @@ orderedSet, ) -class DeezerPlaylistIE(InfoExtractor): +class DeezerBaseInfoExtractor(InfoExtractor): + + def get_data(self, url): + if 'test' not in self._downloader.params: + self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') + + mobj = re.match(self._VALID_URL, url) + id = mobj.group('id') + + webpage = self._download_webpage(url, id) + geoblocking_msg = self._html_search_regex( + r'

(.*?)

', webpage, 'geoblocking message', + default=None) + if geoblocking_msg is not None: + raise ExtractorError( + 'Deezer said: %s' % geoblocking_msg, expected=True) + + data_json = self._search_regex( + (r'__DZR_APP_STATE__\s*=\s*({.+?})\s*', + r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n'), + webpage, 'data JSON') + data = json.loads(data_json) + return id, webpage, data + +class DeezerPlaylistIE(DeezerBaseInfoExtractor): _VALID_URL = r'https?://(?:www\.)?deezer\.com/(../)?playlist/(?P[0-9]+)' _TEST = { 'url': 'http://www.deezer.com/playlist/176747451', @@ -24,25 +48,7 @@ class DeezerPlaylistIE(InfoExtractor): } def _real_extract(self, url): - if 'test' not in self._downloader.params: - self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') - - mobj = re.match(self._VALID_URL, url) - playlist_id = mobj.group('id') - - webpage = self._download_webpage(url, playlist_id) - geoblocking_msg = self._html_search_regex( - r'

(.*?)

', webpage, 'geoblocking message', - default=None) - if geoblocking_msg is not None: - raise ExtractorError( - 'Deezer said: %s' % geoblocking_msg, expected=True) - - data_json = self._search_regex( - (r'__DZR_APP_STATE__\s*=\s*({.+?})\s*', - r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n'), - webpage, 'data JSON') - data = json.loads(data_json) + playlist_id, webpage, data = self.get_data(url) playlist_title = data.get('DATA').get('TITLE') playlist_uploader = data.get('DATA').get('PARENT_USERNAME') From e187464c18eb19e91ba4125a96c934200b9ba30c Mon Sep 17 00:00:00 2001 From: Lucas Date: Sun, 18 Aug 2019 23:41:35 +0200 Subject: [PATCH 5/9] Add album extractor --- youtube_dl/extractor/deezer.py | 59 ++++++++++++++++++++++++++++++ youtube_dl/extractor/extractors.py | 5 ++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index 6a21f2ffa..c5c7d6472 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -85,3 +85,62 @@ def _real_extract(self, url): 'thumbnail': playlist_thumbnail, 'entries': entries, } + +class DeezerAlbumIE(DeezerBaseInfoExtractor): + _VALID_URL = r'https?://(?:www\.)?deezer\.com/(../)?album/(?P[0-9]+)' + _TEST = { + 'url': 'https://www.deezer.com/fr/album/67505622', + 'info_dict': { + 'id': '67505622', + 'title': 'Last Week', + 'uploader': 'Home Brew', + 'thumbnail': r're:^https?://(e-)?cdns-images\.dzcdn\.net/images/cover/.*\.jpg$', + }, + 'playlist_count': 7, + } + + def _real_extract(self, url): + album_id, webpage, data = self.get_data(url) + + album_title = data.get('DATA').get('ALB_TITLE') + album_uploader = data.get('DATA').get('ART_NAME') + album_thumbnail = self._search_regex( + r' Date: Sun, 18 Aug 2019 23:45:03 +0200 Subject: [PATCH 6/9] Warning message is now reported --- youtube_dl/extractor/deezer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index c5c7d6472..7409551d1 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -13,7 +13,7 @@ class DeezerBaseInfoExtractor(InfoExtractor): def get_data(self, url): - if 'test' not in self._downloader.params: + if not self._downloader.params.get('test'): self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') mobj = re.match(self._VALID_URL, url) From 4d3c82222c3249726d9a5b2f06ce2bf9ade9c742 Mon Sep 17 00:00:00 2001 From: Lucas Date: Sun, 18 Aug 2019 23:58:49 +0200 Subject: [PATCH 7/9] flake8 corrections --- youtube_dl/extractor/deezer.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index 7409551d1..bf4204459 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -10,8 +10,8 @@ orderedSet, ) -class DeezerBaseInfoExtractor(InfoExtractor): +class DeezerBaseInfoExtractor(InfoExtractor): def get_data(self, url): if not self._downloader.params.get('test'): self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') @@ -34,6 +34,7 @@ def get_data(self, url): data = json.loads(data_json) return id, webpage, data + class DeezerPlaylistIE(DeezerBaseInfoExtractor): _VALID_URL = r'https?://(?:www\.)?deezer\.com/(../)?playlist/(?P[0-9]+)' _TEST = { @@ -86,6 +87,7 @@ def _real_extract(self, url): 'entries': entries, } + class DeezerAlbumIE(DeezerBaseInfoExtractor): _VALID_URL = r'https?://(?:www\.)?deezer\.com/(../)?album/(?P[0-9]+)' _TEST = { @@ -127,13 +129,12 @@ def _real_extract(self, url): 'uploader_id': s.get('ART_ID'), 'age_limit': 16 if s.get('EXPLICIT_LYRICS') == '1' else 0, 'formats': formats, - 'track' : s.get('SNG_TITLE'), - 'track_number' : int(s.get('TRACK_NUMBER')), - 'track_id' : s.get('SNG_ID'), - 'artist' : album_uploader, - 'album' : album_title, - 'album_artist' : album_uploader, - + 'track': s.get('SNG_TITLE'), + 'track_number': int(s.get('TRACK_NUMBER')), + 'track_id': s.get('SNG_ID'), + 'artist': album_uploader, + 'album': album_title, + 'album_artist': album_uploader, }) return { @@ -143,4 +144,4 @@ def _real_extract(self, url): 'uploader': album_uploader, 'thumbnail': album_thumbnail, 'entries': entries, - } \ No newline at end of file + } From fe7d38c793c8945d82089f7671cbec6af1bf8860 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 19 Aug 2019 00:37:51 +0200 Subject: [PATCH 8/9] Coding convention --- youtube_dl/extractor/deezer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index bf4204459..d0e553f5a 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -130,7 +130,7 @@ def _real_extract(self, url): 'age_limit': 16 if s.get('EXPLICIT_LYRICS') == '1' else 0, 'formats': formats, 'track': s.get('SNG_TITLE'), - 'track_number': int(s.get('TRACK_NUMBER')), + 'track_number': int_or_none(s.get('TRACK_NUMBER')), 'track_id': s.get('SNG_ID'), 'artist': album_uploader, 'album': album_title, From e88689890254ce1f01836d5b98f94ef05e83e99d Mon Sep 17 00:00:00 2001 From: Luc Berge Date: Mon, 2 Sep 2019 19:32:06 +0200 Subject: [PATCH 9/9] Coding convention --- youtube_dl/extractor/deezer.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index d0e553f5a..3031671c1 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -17,9 +17,9 @@ def get_data(self, url): self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') mobj = re.match(self._VALID_URL, url) - id = mobj.group('id') + data_id = mobj.group('id') - webpage = self._download_webpage(url, id) + webpage = self._download_webpage(url, data_id) geoblocking_msg = self._html_search_regex( r'

(.*?)

', webpage, 'geoblocking message', default=None) @@ -32,7 +32,7 @@ def get_data(self, url): r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n'), webpage, 'data JSON') data = json.loads(data_json) - return id, webpage, data + return data_id, webpage, data class DeezerPlaylistIE(DeezerBaseInfoExtractor): @@ -51,17 +51,17 @@ class DeezerPlaylistIE(DeezerBaseInfoExtractor): def _real_extract(self, url): playlist_id, webpage, data = self.get_data(url) - playlist_title = data.get('DATA').get('TITLE') - playlist_uploader = data.get('DATA').get('PARENT_USERNAME') + playlist_title = data.get('DATA', {}).get('TITLE') + playlist_uploader = data.get('DATA', {}).get('PARENT_USERNAME') playlist_thumbnail = self._search_regex( r'