Merge pull request #3969 from shmuelamar/master

fix handle of non-ascii location on redirects issue #3888

requests/sessions.py

@@ -12,7 +12,7 @@ from collections import Mapping
 from datetime import datetime
 
 from .auth import _basic_auth_str
-from .compat import cookielib, OrderedDict, urljoin, urlparse
+from .compat import cookielib, is_py3, OrderedDict, urljoin, urlparse
 from .cookies import (
     cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies)
 from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT
@@ -90,7 +90,16 @@ class SessionRedirectMixin(object):
     def get_redirect_target(self, resp):
         """Receives a Response. Returns a redirect URI or ``None``"""
         if resp.is_redirect:
-            return resp.headers['location']
+            location = resp.headers['location']
+            # Currently the underlying http module on py3 decode headers
+            # in latin1, but empirical evidence suggests that latin1 is very
+            # rarely used with non-ASCII characters in HTTP headers.
+            # It is more likely to get UTF8 header rather than latin1.
+            # This causes incorrect handling of UTF8 encoded location headers.
+            # To solve this, we re-encode the location in latin1.
+            if is_py3:
+                location = location.encode('latin1')
+            return to_native_string(location, 'utf8')
         return None
 
     def resolve_redirects(self, resp, req, stream=False, timeout=None,

tests/test_lowlevel.py

@@ -3,7 +3,6 @@
 import pytest
 import threading
 import requests
-from requests.compat import quote, is_py3
 
 from tests.testserver.server import Server, consume_socket_content
 
@@ -208,8 +207,7 @@ def test_use_proxy_from_environment(httpbin, var, scheme):
 
 def test_redirect_rfc1808_to_non_ascii_location():
     path = u'š'
-    expected_path = quote(path.encode('utf8')).encode('ascii')
-    expected_path_py3 = b'%C3%85%C2%A1'
+    expected_path = b'%C5%A1'
     redirect_request = []  # stores the second request to the server
 
     def redirect_resp_handler(sock):
@@ -233,11 +231,7 @@ def test_redirect_rfc1808_to_non_ascii_location():
         assert r.status_code == 200
         assert len(r.history) == 1
         assert r.history[0].status_code == 301
-
-        # currently Python3 not handling non-ASCII redirects (issue #3888)
-        if is_py3:
-            assert redirect_request[0].startswith(b'GET /' + expected_path_py3 + b' HTTP/1.1')
-        else:
-            assert redirect_request[0].startswith(b'GET /' + expected_path + b' HTTP/1.1')
+        assert redirect_request[0].startswith(b'GET /' + expected_path + b' HTTP/1.1')
+        assert r.url == u'{0}/{1}'.format(url, expected_path.decode('ascii'))
 
         close_server.set()