diff --git a/.travis.yml b/.travis.yml
index f41e11137f..b655738ec7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,10 @@ sudo: false
 env:
   - YTDL_TEST_SET=core
   - YTDL_TEST_SET=download
+matrix:
+  fast_finish: true
+  allow_failures:
+    - env: YTDL_TEST_SET=download
 script: ./devscripts/run_tests.sh
 notifications:
   email: