Merge branch 'pr-crashfix_compat_urllib_unquote' of https://github.com/atomicdryad/youtube-dl into atomicdryad-pr-crashfix_compat_urllib_unquote

test/test_compat.py

@@ -14,6 +14,7 @@ from youtube_dl.utils import get_filesystem_encoding
 from youtube_dl.compat import (
     compat_getenv,
     compat_expanduser,
+    compat_urllib_parse_unquote,
 )
 
 
@@ -42,5 +43,24 @@ class TestCompat(unittest.TestCase):
             dir(youtube_dl.compat))) - set(['unicode_literals'])
         self.assertEqual(all_names, sorted(present_names))
 
+    def test_compat_urllib_parse_unquote(self):
+        test_strings = [
+            ['''''', ''''''],
+            ['''津波''', '''%E6%B4%A5%E6%B3%A2'''],
+            ['''津波''', str('%E6%B4%A5%E6%B3%A2')],
+            ['''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
+%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''',
+             '''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
+%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''],
+            ['''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''',
+             '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%''']
+        ]
+        for test in test_strings:
+            strutf = test[0]
+            strurlenc = test[1]
+            strurldec = compat_urllib_parse_unquote(strurlenc)
+            self.assertEqual(strutf, strurldec)
+            self.assertEqual(strutf, compat_urllib_parse_unquote(strurlenc))
+
 if __name__ == '__main__':
     unittest.main()

youtube_dl/compat.py

@@ -77,40 +77,73 @@ except ImportError:
 try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
 except ImportError:
-    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
-        if string == '':
+    def compat_urllib_parse_unquote_to_bytes(string):
+        """unquote_to_bytes('abc%20def') -> b'abc def'."""
+        # Note: strings are encoded as UTF-8. This is only an issue if it contains
+        # unescaped non-ASCII characters, which URIs should not.
+        if not string:
+            # Is it a string-like object?
+            string.split
+            return b''
+        if isinstance(string, str):
+            string = string.encode('utf-8')
+            # string = encode('utf-8')
+
+        # python3 -> 2: must implicitly convert to bits
+        bits = bytes(string).split(b'%')
+
+        if len(bits) == 1:
             return string
-        res = string.split('%')
-        if len(res) == 1:
+        res = [bits[0]]
+        append = res.append
+
+        for item in bits[1:]:
+            if item == '':
+                append(b'%')
+                continue
+            try:
+                append(item[:2].decode('hex'))
+                append(item[2:])
+            except:
+                append(b'%')
+                append(item)
+        return b''.join(res)
+
+    compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)')
+
+    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+        """Replace %xx escapes by their single-character equivalent. The optional
+        encoding and errors parameters specify how to decode percent-encoded
+        sequences into Unicode characters, as accepted by the bytes.decode()
+        method.
+        By default, percent-encoded sequences are decoded with UTF-8, and invalid
+        sequences are replaced by a placeholder character.
+
+        unquote('abc%20def') -> 'abc def'.
+        """
+
+        if '%' not in string:
+            string.split
             return string
         if encoding is None:
             encoding = 'utf-8'
         if errors is None:
             errors = 'replace'
-        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
-        pct_sequence = b''
-        string = res[0]
-        for item in res[1:]:
-            try:
-                if not item:
-                    raise ValueError
-                pct_sequence += item[:2].decode('hex')
-                rest = item[2:]
-                if not rest:
-                    # This segment was just a single percent-encoded character.
-                    # May be part of a sequence of code units, so delay decoding.
-                    # (Stored in pct_sequence).
-                    continue
-            except ValueError:
-                rest = '%' + item
-            # Encountered non-percent-encoded characters. Flush the current
-            # pct_sequence.
-            string += pct_sequence.decode(encoding, errors) + rest
-            pct_sequence = b''
-        if pct_sequence:
-            # Flush the final pct_sequence
-            string += pct_sequence.decode(encoding, errors)
-        return string
+
+        bits = compat_urllib_parse_asciire.split(string)
+        res = [bits[0]]
+        append = res.append
+        for i in range(1, len(bits), 2):
+            foo = compat_urllib_parse_unquote_to_bytes(bits[i])
+            foo = foo.decode(encoding, errors)
+            append(foo)
+
+            if bits[i + 1]:
+                bar = bits[i + 1]
+                if not isinstance(bar, unicode):
+                    bar = bar.decode('utf-8')
+                append(bar)
+        return ''.join(res)
 
 try:
     compat_str = unicode  # Python 2
@@ -421,7 +454,9 @@ __all__ = [
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',
+    'compat_urllib_parse_asciire',
     'compat_urllib_parse_unquote',
+    'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',
     'compat_urlparse',

youtube_dl/extractor/generic.py

@@ -1115,7 +1115,7 @@ class GenericIE(InfoExtractor):
         # Sometimes embedded video player is hidden behind percent encoding
         # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
         # Unescaping the whole page allows to handle those cases in a generic way
-        webpage = compat_urllib_parse.unquote(webpage)
+        webpage = compat_urllib_parse_unquote(webpage)
 
         # it's tempting to parse this further, but you would
         # have to take into account all the variations like