Merge fdb694eca1
into 668332b973
This commit is contained in:
commit
386e52710c
|
@ -20,6 +20,7 @@ from youtube_dl.compat import (
|
||||||
from youtube_dl.utils import (
|
from youtube_dl.utils import (
|
||||||
IDENTITY,
|
IDENTITY,
|
||||||
preferredencoding,
|
preferredencoding,
|
||||||
|
variadic,
|
||||||
write_string,
|
write_string,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -66,7 +67,7 @@ def report_warning(message):
|
||||||
class FakeYDL(YoutubeDL):
|
class FakeYDL(YoutubeDL):
|
||||||
def __init__(self, override=None):
|
def __init__(self, override=None):
|
||||||
# Different instances of the downloader can't share the same dictionary
|
# Different instances of the downloader can't share the same dictionary
|
||||||
# some test set the "sublang" parameter, which would break the md5 checks.
|
# some tests set the "sublang" parameter, which would break the md5 checks.
|
||||||
params = get_params(override=override)
|
params = get_params(override=override)
|
||||||
super(FakeYDL, self).__init__(params, auto_init=False)
|
super(FakeYDL, self).__init__(params, auto_init=False)
|
||||||
self.result = []
|
self.result = []
|
||||||
|
@ -83,13 +84,7 @@ class FakeYDL(YoutubeDL):
|
||||||
|
|
||||||
def expect_warning(self, regex):
|
def expect_warning(self, regex):
|
||||||
# Silence an expected warning matching a regex
|
# Silence an expected warning matching a regex
|
||||||
old_report_warning = self.report_warning
|
expect_warnings(self, regex)
|
||||||
|
|
||||||
def report_warning(self, message):
|
|
||||||
if re.match(regex, message):
|
|
||||||
return
|
|
||||||
old_report_warning(message)
|
|
||||||
self.report_warning = types.MethodType(report_warning, self)
|
|
||||||
|
|
||||||
|
|
||||||
class FakeLogger(object):
|
class FakeLogger(object):
|
||||||
|
@ -285,12 +280,14 @@ def assertEqual(self, got, expected, msg=None):
|
||||||
|
|
||||||
def expect_warnings(ydl, warnings_re):
|
def expect_warnings(ydl, warnings_re):
|
||||||
real_warning = ydl.report_warning
|
real_warning = ydl.report_warning
|
||||||
|
# to facilitate matching, don't prettify messages
|
||||||
|
ydl.params['no_color'] = True
|
||||||
|
|
||||||
def _report_warning(w):
|
def _report_warning(self, w, *args, **kwargs):
|
||||||
if not any(re.search(w_re, w) for w_re in warnings_re):
|
if not any(re.search(w_re, w) for w_re in variadic(warnings_re)):
|
||||||
real_warning(w)
|
real_warning(w, *args, **kwargs)
|
||||||
|
|
||||||
ydl.report_warning = _report_warning
|
ydl.report_warning = types.MethodType(_report_warning, ydl)
|
||||||
|
|
||||||
|
|
||||||
def http_server_port(httpd):
|
def http_server_port(httpd):
|
||||||
|
|
|
@ -21,13 +21,17 @@ from test.helper import (
|
||||||
|
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import itertools
|
||||||
import json
|
import json
|
||||||
import socket
|
import socket
|
||||||
|
import re
|
||||||
|
|
||||||
import youtube_dl.YoutubeDL
|
import youtube_dl.YoutubeDL
|
||||||
from youtube_dl.compat import (
|
from youtube_dl.compat import (
|
||||||
|
compat_filter as filter,
|
||||||
compat_http_client,
|
compat_http_client,
|
||||||
compat_HTTPError,
|
compat_HTTPError,
|
||||||
|
compat_map as map,
|
||||||
compat_open as open,
|
compat_open as open,
|
||||||
compat_urllib_error,
|
compat_urllib_error,
|
||||||
)
|
)
|
||||||
|
@ -36,9 +40,11 @@ from youtube_dl.utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
error_to_compat_str,
|
error_to_compat_str,
|
||||||
format_bytes,
|
format_bytes,
|
||||||
|
std_headers,
|
||||||
UnavailableVideoError,
|
UnavailableVideoError,
|
||||||
)
|
)
|
||||||
from youtube_dl.extractor import get_info_extractor
|
from youtube_dl.extractor import get_info_extractor
|
||||||
|
from youtube_dl.downloader.common import FileDownloader
|
||||||
|
|
||||||
RETRIES = 3
|
RETRIES = 3
|
||||||
|
|
||||||
|
@ -49,7 +55,7 @@ class YoutubeDL(youtube_dl.YoutubeDL):
|
||||||
self.processed_info_dicts = []
|
self.processed_info_dicts = []
|
||||||
super(YoutubeDL, self).__init__(*args, **kwargs)
|
super(YoutubeDL, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
def report_warning(self, message):
|
def report_warning(self, message, *args, **kwargs):
|
||||||
# Don't accept warnings during tests
|
# Don't accept warnings during tests
|
||||||
raise ExtractorError(message)
|
raise ExtractorError(message)
|
||||||
|
|
||||||
|
@ -58,9 +64,10 @@ class YoutubeDL(youtube_dl.YoutubeDL):
|
||||||
return super(YoutubeDL, self).process_info(info_dict)
|
return super(YoutubeDL, self).process_info(info_dict)
|
||||||
|
|
||||||
|
|
||||||
def _file_md5(fn):
|
def _file_md5(fn, length=None):
|
||||||
with open(fn, 'rb') as f:
|
with open(fn, 'rb') as f:
|
||||||
return hashlib.md5(f.read()).hexdigest()
|
return hashlib.md5(
|
||||||
|
f.read() if length is None else f.read(length)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
defs = gettestcases()
|
defs = gettestcases()
|
||||||
|
@ -85,6 +92,13 @@ class TestDownload(unittest.TestCase):
|
||||||
strclass(self.__class__),
|
strclass(self.__class__),
|
||||||
' [%s]' % add_ie if add_ie else '')
|
' [%s]' % add_ie if add_ie else '')
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def addTest(cls, test_method, test_method_name, add_ie):
|
||||||
|
test_method.__name__ = str(test_method_name)
|
||||||
|
test_method.add_ie = add_ie
|
||||||
|
setattr(cls, test_method.__name__, test_method)
|
||||||
|
del test_method
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
self.defs = defs
|
self.defs = defs
|
||||||
|
|
||||||
|
@ -129,6 +143,17 @@ def generator(test_case, tname):
|
||||||
else test_case.get('playlist_mincount'))
|
else test_case.get('playlist_mincount'))
|
||||||
params.setdefault('skip_download', True)
|
params.setdefault('skip_download', True)
|
||||||
|
|
||||||
|
if 'user_agent' in params:
|
||||||
|
std_headers['User-Agent'] = params['user_agent']
|
||||||
|
|
||||||
|
if 'referer' in params:
|
||||||
|
std_headers['Referer'] = params['referer']
|
||||||
|
|
||||||
|
for h in params.get('headers', []):
|
||||||
|
h = h.split(':', 1)
|
||||||
|
if len(h) > 1:
|
||||||
|
std_headers[h[0]] = h[1]
|
||||||
|
|
||||||
ydl = YoutubeDL(params, auto_init=False)
|
ydl = YoutubeDL(params, auto_init=False)
|
||||||
ydl.add_default_info_extractors()
|
ydl.add_default_info_extractors()
|
||||||
finished_hook_called = set()
|
finished_hook_called = set()
|
||||||
|
@ -155,8 +180,7 @@ def generator(test_case, tname):
|
||||||
|
|
||||||
try_rm_tcs_files()
|
try_rm_tcs_files()
|
||||||
try:
|
try:
|
||||||
try_num = 1
|
for try_num in itertools.count(1):
|
||||||
while True:
|
|
||||||
try:
|
try:
|
||||||
# We're not using .download here since that is just a shim
|
# We're not using .download here since that is just a shim
|
||||||
# for outside error handling, and returns the exit code
|
# for outside error handling, and returns the exit code
|
||||||
|
@ -165,7 +189,7 @@ def generator(test_case, tname):
|
||||||
test_case['url'],
|
test_case['url'],
|
||||||
force_generic_extractor=params.get('force_generic_extractor', False))
|
force_generic_extractor=params.get('force_generic_extractor', False))
|
||||||
except (DownloadError, ExtractorError) as err:
|
except (DownloadError, ExtractorError) as err:
|
||||||
# Check if the exception is not a network related one
|
# Retry, or raise if the exception is not network-related
|
||||||
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
|
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
|
||||||
msg = getattr(err, 'msg', error_to_compat_str(err))
|
msg = getattr(err, 'msg', error_to_compat_str(err))
|
||||||
err.msg = '%s (%s)' % (msg, tname, )
|
err.msg = '%s (%s)' % (msg, tname, )
|
||||||
|
@ -176,8 +200,6 @@ def generator(test_case, tname):
|
||||||
return
|
return
|
||||||
|
|
||||||
print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
|
print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
|
||||||
|
|
||||||
try_num += 1
|
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
|
|
||||||
|
@ -249,7 +271,7 @@ def generator(test_case, tname):
|
||||||
(tc_filename, format_bytes(expected_minsize),
|
(tc_filename, format_bytes(expected_minsize),
|
||||||
format_bytes(got_fsize)))
|
format_bytes(got_fsize)))
|
||||||
if 'md5' in tc:
|
if 'md5' in tc:
|
||||||
md5_for_file = _file_md5(tc_filename)
|
md5_for_file = _file_md5(tc_filename) if not params.get('test') else _file_md5(tc_filename, FileDownloader._TEST_FILE_SIZE)
|
||||||
self.assertEqual(tc['md5'], md5_for_file)
|
self.assertEqual(tc['md5'], md5_for_file)
|
||||||
# Finally, check test cases' data again but this time against
|
# Finally, check test cases' data again but this time against
|
||||||
# extracted data from info JSON file written during processing
|
# extracted data from info JSON file written during processing
|
||||||
|
@ -279,12 +301,42 @@ for n, test_case in enumerate(defs):
|
||||||
tname = 'test_%s_%d' % (test_case['name'], i)
|
tname = 'test_%s_%d' % (test_case['name'], i)
|
||||||
i += 1
|
i += 1
|
||||||
test_method = generator(test_case, tname)
|
test_method = generator(test_case, tname)
|
||||||
test_method.__name__ = str(tname)
|
ie_list = ','.join(test_case.get('add_ie', []))
|
||||||
ie_list = test_case.get('add_ie')
|
TestDownload.addTest(test_method, tname, ie_list)
|
||||||
test_method.add_ie = ie_list and ','.join(ie_list)
|
|
||||||
setattr(TestDownload, test_method.__name__, test_method)
|
|
||||||
del test_method
|
|
||||||
|
|
||||||
|
|
||||||
|
def tests_for_ie(ie_key):
|
||||||
|
return filter(
|
||||||
|
lambda a: callable(getattr(TestDownload, a, None)),
|
||||||
|
filter(lambda a: re.match(r'test_%s(?:_\d+)?$' % ie_key, a),
|
||||||
|
dir(TestDownload)))
|
||||||
|
|
||||||
|
|
||||||
|
def gen_test_suite(ie_key):
|
||||||
|
def test_all(self):
|
||||||
|
print(self)
|
||||||
|
suite = unittest.TestSuite(
|
||||||
|
map(TestDownload, tests_for_ie(ie_key)))
|
||||||
|
result = self.defaultTestResult()
|
||||||
|
suite.run(result)
|
||||||
|
print('Errors: %d\t Failures: %d\tSkipped: %d' %
|
||||||
|
tuple(map(len, (result.errors, result.failures, result.skipped))))
|
||||||
|
print('Expected failures: %d\tUnexpected successes: %d' %
|
||||||
|
tuple(map(len, (result.expectedFailures, result.unexpectedSuccesses))))
|
||||||
|
return result
|
||||||
|
|
||||||
|
return test_all
|
||||||
|
|
||||||
|
|
||||||
|
for ie_key in set(
|
||||||
|
map(lambda a: a[5:],
|
||||||
|
filter(
|
||||||
|
lambda x: callable(getattr(TestDownload, x, None)),
|
||||||
|
filter(
|
||||||
|
lambda t: re.match(r"test_.+(?<!(?:_all|.._\d|._\d\d|_\d\d\d))$", t),
|
||||||
|
dir(TestDownload))))):
|
||||||
|
test_all = gen_test_suite(ie_key)
|
||||||
|
TestDownload.addTest(test_all, 'test_%s_all' % ie_key, 'Test all: %s' % ie_key)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
Loading…
Reference in New Issue