Merge pull request #3963 from shmuelamar/master

fix unicode decode error on py2 when handling redirect without scheme

AUTHORS.rst

@@ -179,4 +179,4 @@ Patches and Suggestions
 - Matt Kohl (`@mattkohl <https://github.com/mattkohl>`_)
 - Jonathan Vanasco (`@jvanasco <https://github.com/jvanasco>`_)
 - David Fontenot (`@davidfontenot <https://github.com/davidfontenot>`_)
-
+- Shmuel Amar (`@shmuelamar <https://github.com/shmuelamar>`_)

requests/sessions.py

@@ -122,7 +122,7 @@ class SessionRedirectMixin(object):
             # Handle redirection without scheme (see: RFC 1808 Section 4)
             if url.startswith('//'):
                 parsed_rurl = urlparse(resp.url)
-                url = '%s:%s' % (parsed_rurl.scheme, url)
+                url = '%s:%s' % (to_native_string(parsed_rurl.scheme), url)
 
             # The scheme should be lower case...
             parsed = urlparse(url)

tests/test_lowlevel.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 
-import os
 import pytest
 import threading
 import requests
+from requests.compat import quote, is_py3
 
 from tests.testserver.server import Server, consume_socket_content
 
@@ -204,3 +204,40 @@ def test_use_proxy_from_environment(httpbin, var, scheme):
 
         # it had actual content (not checking for SOCKS protocol for now)
         assert len(fake_proxy.handler_results[0]) > 0
+
+
+def test_redirect_rfc1808_to_non_ascii_location():
+    path = u'š'
+    expected_path = quote(path.encode('utf8')).encode('ascii')
+    expected_path_py3 = b'%C3%85%C2%A1'
+    redirect_request = []  # stores the second request to the server
+
+    def redirect_resp_handler(sock):
+        consume_socket_content(sock, timeout=0.5)
+        location = u'//{0}:{1}/{2}'.format(host, port, path)
+        sock.send(
+            b'HTTP/1.1 301 Moved Permanently\r\n'
+            b'Content-Length: 0\r\n'
+            b'Location: ' + location.encode('utf8') + b'\r\n'
+            b'\r\n'
+        )
+        redirect_request.append(consume_socket_content(sock, timeout=0.5))
+        sock.send(b'HTTP/1.1 200 OK\r\n\r\n')
+
+    close_server = threading.Event()
+    server = Server(redirect_resp_handler, wait_to_close_event=close_server)
+
+    with server as (host, port):
+        url = u'http://{0}:{1}'.format(host, port)
+        r = requests.get(url=url, allow_redirects=True)
+        assert r.status_code == 200
+        assert len(r.history) == 1
+        assert r.history[0].status_code == 301
+
+        # currently Python3 not handling non-ASCII redirects (issue #3888)
+        if is_py3:
+            assert redirect_request[0].startswith(b'GET /' + expected_path_py3 + b' HTTP/1.1')
+        else:
+            assert redirect_request[0].startswith(b'GET /' + expected_path + b' HTTP/1.1')
+
+        close_server.set()