diff --git a/AUTHORS.rst b/AUTHORS.rst index 289f1aef..9e92a959 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -169,3 +169,4 @@ Patches and Suggestions - Nate Prewitt (`@nateprewitt `_) - Maik Himstedt - Michael Hunsinger +- Brian Bamsch (`@bbamsch `_) diff --git a/requests/_internal_utils.py b/requests/_internal_utils.py new file mode 100644 index 00000000..87e2ca00 --- /dev/null +++ b/requests/_internal_utils.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +""" +requests._internal_utils +~~~~~~~~~~~~~~ + +Provides utility functions that are consumed internally by Requests +which depend on extremely few external helpers (such as compat) +""" + +from .compat import is_py2, builtin_str + + +def to_native_string(string, encoding='ascii'): + """Given a string object, regardless of type, returns a representation of + that string in the native string type, encoding and decoding where + necessary. This assumes ASCII unless told otherwise. + """ + if isinstance(string, builtin_str): + out = string + else: + if is_py2: + out = string.encode(encoding) + else: + out = string.decode(encoding) + + return out diff --git a/requests/auth.py b/requests/auth.py index 49bcb24a..d1196daa 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -17,7 +17,8 @@ from base64 import b64encode from .compat import urlparse, str from .cookies import extract_cookies_to_jar -from .utils import parse_dict_header, to_native_string +from ._internal_utils import to_native_string +from .utils import parse_dict_header from .status_codes import codes CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' diff --git a/requests/cookies.py b/requests/cookies.py index 41a2fde1..856fd45e 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -13,6 +13,8 @@ import copy import time import calendar import collections + +from ._internal_utils import to_native_string from .compat import cookielib, urlparse, urlunparse, Morsel try: @@ -55,7 +57,7 @@ class MockRequest(object): if not self._r.headers.get('Host'): return self._r.url # If they did set it, retrieve it and reconstruct the expected domain - host = self._r.headers['Host'] + host = to_native_string(self._r.headers['Host'], encoding='utf-8') parsed = urlparse(self._r.url) # Reconstruct the URL as we expect it return urlunparse([ diff --git a/requests/models.py b/requests/models.py index ea25e356..57ab7700 100644 --- a/requests/models.py +++ b/requests/models.py @@ -29,11 +29,11 @@ from .packages.urllib3.exceptions import ( from .exceptions import ( HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, ContentDecodingError, ConnectionError, StreamConsumedError) +from ._internal_utils import to_native_string from .utils import ( guess_filename, get_auth_from_url, requote_uri, stream_decode_response_unicode, to_key_val_list, parse_header_links, - iter_slices, guess_json_utf, super_len, to_native_string, - check_header_validity) + iter_slices, guess_json_utf, super_len, check_header_validity) from .compat import ( cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO, is_py2, chardet, builtin_str, basestring) diff --git a/requests/sessions.py b/requests/sessions.py index b05150a4..971575b2 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -17,7 +17,8 @@ from .cookies import ( cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook -from .utils import to_key_val_list, default_headers, to_native_string +from ._internal_utils import to_native_string +from .utils import to_key_val_list, default_headers from .exceptions import ( TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) from .packages.urllib3._collections import RecentlyUsedContainer diff --git a/requests/utils.py b/requests/utils.py index 4b2126e6..3b43ad7d 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -20,10 +20,11 @@ import warnings from . import __version__ from . import certs +# to_native_string is unused here, but imported here for backwards compatibility +from ._internal_utils import to_native_string from .compat import parse_http_list as _parse_list_header -from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, is_py2, - builtin_str, getproxies, proxy_bypass, urlunparse, - basestring) +from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, + getproxies, proxy_bypass, urlunparse, basestring) from .cookies import RequestsCookieJar, cookiejar_from_dict from .structures import CaseInsensitiveDict from .exceptions import InvalidURL, InvalidHeader, FileModeWarning @@ -770,22 +771,6 @@ def get_auth_from_url(url): return auth -def to_native_string(string, encoding='ascii'): - """Given a string object, regardless of type, returns a representation of - that string in the native string type, encoding and decoding where - necessary. This assumes ASCII unless told otherwise. - """ - if isinstance(string, builtin_str): - out = string - else: - if is_py2: - out = string.encode(encoding) - else: - out = string.decode(encoding) - - return out - - # Moved outside of function to avoid recompile every call _CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$') _CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$') diff --git a/tests/test_requests.py b/tests/test_requests.py index eaaf90c1..b6e8b99e 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -858,6 +858,16 @@ class TestRequests: prep = s.prepare_request(req) assert prep.url == "https://httpbin.org/" + def test_request_with_bytestring_host(self): + s = requests.Session() + resp = s.request( + 'GET', + 'http://httpbin.org/cookies/set?cookie=value', + allow_redirects=False, + headers={'Host': b'httpbin.org'} + ) + assert resp.cookies.get('cookie') == 'value' + def test_links(self): r = requests.Response() r.headers = {