Merge pull request #3598 from bbamsch/bytestringHostFix

Issue #3597 - Avoid bytestring/str hodgepodge when resolving cookie from Request URL

AUTHORS.rst

@@ -169,3 +169,4 @@ Patches and Suggestions
 - Nate Prewitt <nate.prewitt@gmail.com> (`@nateprewitt <https://github.com/nateprewitt>`_)
 - Maik Himstedt
 - Michael Hunsinger
+- Brian Bamsch <bbamsch32@gmail.com> (`@bbamsch <https://github.com/bbamsch>`_)

requests/_internal_utils.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+
+"""
+requests._internal_utils
+~~~~~~~~~~~~~~
+
+Provides utility functions that are consumed internally by Requests
+which depend on extremely few external helpers (such as compat)
+"""
+
+from .compat import is_py2, builtin_str
+
+
+def to_native_string(string, encoding='ascii'):
+    """Given a string object, regardless of type, returns a representation of
+    that string in the native string type, encoding and decoding where
+    necessary. This assumes ASCII unless told otherwise.
+    """
+    if isinstance(string, builtin_str):
+        out = string
+    else:
+        if is_py2:
+            out = string.encode(encoding)
+        else:
+            out = string.decode(encoding)
+
+    return out

requests/auth.py

@@ -17,7 +17,8 @@ from base64 import b64encode
 
 from .compat import urlparse, str
 from .cookies import extract_cookies_to_jar
-from .utils import parse_dict_header, to_native_string
+from ._internal_utils import to_native_string
+from .utils import parse_dict_header
 from .status_codes import codes
 
 CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'

requests/cookies.py

@@ -13,6 +13,8 @@ import copy
 import time
 import calendar
 import collections
+
+from ._internal_utils import to_native_string
 from .compat import cookielib, urlparse, urlunparse, Morsel
 
 try:
@@ -55,7 +57,7 @@ class MockRequest(object):
         if not self._r.headers.get('Host'):
             return self._r.url
         # If they did set it, retrieve it and reconstruct the expected domain
-        host = self._r.headers['Host']
+        host = to_native_string(self._r.headers['Host'], encoding='utf-8')
         parsed = urlparse(self._r.url)
         # Reconstruct the URL as we expect it
         return urlunparse([

requests/models.py

@@ -29,11 +29,11 @@ from .packages.urllib3.exceptions import (
 from .exceptions import (
     HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError,
     ContentDecodingError, ConnectionError, StreamConsumedError)
+from ._internal_utils import to_native_string
 from .utils import (
     guess_filename, get_auth_from_url, requote_uri,
     stream_decode_response_unicode, to_key_val_list, parse_header_links,
-    iter_slices, guess_json_utf, super_len, to_native_string,
-    check_header_validity)
+    iter_slices, guess_json_utf, super_len, check_header_validity)
 from .compat import (
     cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO,
     is_py2, chardet, builtin_str, basestring)

requests/sessions.py

@@ -17,7 +17,8 @@ from .cookies import (
     cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies)
 from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT
 from .hooks import default_hooks, dispatch_hook
-from .utils import to_key_val_list, default_headers, to_native_string
+from ._internal_utils import to_native_string
+from .utils import to_key_val_list, default_headers
 from .exceptions import (
     TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError)
 from .packages.urllib3._collections import RecentlyUsedContainer

requests/utils.py

@@ -20,10 +20,11 @@ import warnings
 
 from . import __version__
 from . import certs
+# to_native_string is unused here, but imported here for backwards compatibility
+from ._internal_utils import to_native_string
 from .compat import parse_http_list as _parse_list_header
-from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, is_py2,
-                     builtin_str, getproxies, proxy_bypass, urlunparse,
-                     basestring)
+from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote,
+                     getproxies, proxy_bypass, urlunparse, basestring)
 from .cookies import RequestsCookieJar, cookiejar_from_dict
 from .structures import CaseInsensitiveDict
 from .exceptions import InvalidURL, InvalidHeader, FileModeWarning
@@ -770,22 +771,6 @@ def get_auth_from_url(url):
     return auth
 
 
-def to_native_string(string, encoding='ascii'):
-    """Given a string object, regardless of type, returns a representation of
-    that string in the native string type, encoding and decoding where
-    necessary. This assumes ASCII unless told otherwise.
-    """
-    if isinstance(string, builtin_str):
-        out = string
-    else:
-        if is_py2:
-            out = string.encode(encoding)
-        else:
-            out = string.decode(encoding)
-
-    return out
-
-
 # Moved outside of function to avoid recompile every call
 _CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$')
 _CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$')

tests/test_requests.py

@@ -858,6 +858,16 @@ class TestRequests:
         prep = s.prepare_request(req)
         assert prep.url == "https://httpbin.org/"
 
+    def test_request_with_bytestring_host(self):
+        s = requests.Session()
+        resp = s.request(
+            'GET',
+            'http://httpbin.org/cookies/set?cookie=value',
+            allow_redirects=False,
+            headers={'Host': b'httpbin.org'}
+        )
+        assert resp.cookies.get('cookie') == 'value'
+
     def test_links(self):
         r = requests.Response()
         r.headers = {