From b28531f4bb43b401e3e37e2da622aee21002e5b2 Mon Sep 17 00:00:00 2001 From: Brian Bamsch Date: Sun, 25 Sep 2016 21:50:29 -0700 Subject: [PATCH 1/7] Add test for request when Host header is bytestring Request should successfully set cookie in response, fail if cookie fails to set as will occur in current state when cookie set tries lookup of domain --- tests/test_requests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_requests.py b/tests/test_requests.py index eaaf90c1..b6e8b99e 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -858,6 +858,16 @@ class TestRequests: prep = s.prepare_request(req) assert prep.url == "https://httpbin.org/" + def test_request_with_bytestring_host(self): + s = requests.Session() + resp = s.request( + 'GET', + 'http://httpbin.org/cookies/set?cookie=value', + allow_redirects=False, + headers={'Host': b'httpbin.org'} + ) + assert resp.cookies.get('cookie') == 'value' + def test_links(self): r = requests.Response() r.headers = { From 01e405c6116c94d92057f7aaea58d51d1d7f90a7 Mon Sep 17 00:00:00 2001 From: Brian Bamsch Date: Sun, 25 Sep 2016 22:18:45 -0700 Subject: [PATCH 2/7] Add patch to avoid bytestring/str hodgepodge When patching Host header into URL, verify that host type matches urlparse return type before putting it all back together --- requests/cookies.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requests/cookies.py b/requests/cookies.py index 41a2fde1..a64ea38e 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -57,6 +57,12 @@ class MockRequest(object): # If they did set it, retrieve it and reconstruct the expected domain host = self._r.headers['Host'] parsed = urlparse(self._r.url) + + # If parsed url is str type, ensure that host is also str type + if isinstance(parsed.scheme, str) and not isinstance(host, str)\ + and isinstance(host, bytes): + host = host.decode('ascii') + # Reconstruct the URL as we expect it return urlunparse([ parsed.scheme, host, parsed.path, parsed.params, parsed.query, From 550dc190992e227c1b568dcd7a85a81ab7b7a3c0 Mon Sep 17 00:00:00 2001 From: Brian Bamsch Date: Sun, 25 Sep 2016 22:20:55 -0700 Subject: [PATCH 3/7] Add line to AUTHORS Credit where credit is due :smile: --- AUTHORS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 289f1aef..9e92a959 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -169,3 +169,4 @@ Patches and Suggestions - Nate Prewitt (`@nateprewitt `_) - Maik Himstedt - Michael Hunsinger +- Brian Bamsch (`@bbamsch `_) From 2af059797a33a1ee2132ed13e456413c18bc7aa2 Mon Sep 17 00:00:00 2001 From: Brian Bamsch Date: Mon, 26 Sep 2016 09:11:18 -0700 Subject: [PATCH 4/7] Adjust patch to utilize requests utility functions Use the utility class provided by requests to do native string transformation rather than recreating the wheel. --- requests/cookies.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/requests/cookies.py b/requests/cookies.py index a64ea38e..4a6bedd0 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -13,6 +13,8 @@ import copy import time import calendar import collections + +from . import utils from .compat import cookielib, urlparse, urlunparse, Morsel try: @@ -55,14 +57,8 @@ class MockRequest(object): if not self._r.headers.get('Host'): return self._r.url # If they did set it, retrieve it and reconstruct the expected domain - host = self._r.headers['Host'] + host = utils.to_native_string(self._r.headers['Host'], encoding='utf-8') parsed = urlparse(self._r.url) - - # If parsed url is str type, ensure that host is also str type - if isinstance(parsed.scheme, str) and not isinstance(host, str)\ - and isinstance(host, bytes): - host = host.decode('ascii') - # Reconstruct the URL as we expect it return urlunparse([ parsed.scheme, host, parsed.path, parsed.params, parsed.query, From f002b73026bdc136654ba545296898af9a88007a Mon Sep 17 00:00:00 2001 From: Brian Bamsch Date: Mon, 26 Sep 2016 21:41:01 -0700 Subject: [PATCH 5/7] Move to_native_string to _internal_utils.py to avoid circular dependency --- requests/_internal_utils.py | 17 +++++++++++++++++ requests/cookies.py | 4 ++-- requests/utils.py | 23 ++++------------------- 3 files changed, 23 insertions(+), 21 deletions(-) create mode 100644 requests/_internal_utils.py diff --git a/requests/_internal_utils.py b/requests/_internal_utils.py new file mode 100644 index 00000000..0456017c --- /dev/null +++ b/requests/_internal_utils.py @@ -0,0 +1,17 @@ +from .compat import is_py2, builtin_str + + +def to_native_string(string, encoding='ascii'): + """Given a string object, regardless of type, returns a representation of + that string in the native string type, encoding and decoding where + necessary. This assumes ASCII unless told otherwise. + """ + if isinstance(string, builtin_str): + out = string + else: + if is_py2: + out = string.encode(encoding) + else: + out = string.decode(encoding) + + return out diff --git a/requests/cookies.py b/requests/cookies.py index 4a6bedd0..856fd45e 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -14,7 +14,7 @@ import time import calendar import collections -from . import utils +from ._internal_utils import to_native_string from .compat import cookielib, urlparse, urlunparse, Morsel try: @@ -57,7 +57,7 @@ class MockRequest(object): if not self._r.headers.get('Host'): return self._r.url # If they did set it, retrieve it and reconstruct the expected domain - host = utils.to_native_string(self._r.headers['Host'], encoding='utf-8') + host = to_native_string(self._r.headers['Host'], encoding='utf-8') parsed = urlparse(self._r.url) # Reconstruct the URL as we expect it return urlunparse([ diff --git a/requests/utils.py b/requests/utils.py index 4b2126e6..f7d4e37f 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -20,10 +20,11 @@ import warnings from . import __version__ from . import certs +# noinspection PyUnresolvedReferences +from ._internal_utils import to_native_string from .compat import parse_http_list as _parse_list_header -from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, is_py2, - builtin_str, getproxies, proxy_bypass, urlunparse, - basestring) +from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, + getproxies, proxy_bypass, urlunparse, basestring) from .cookies import RequestsCookieJar, cookiejar_from_dict from .structures import CaseInsensitiveDict from .exceptions import InvalidURL, InvalidHeader, FileModeWarning @@ -770,22 +771,6 @@ def get_auth_from_url(url): return auth -def to_native_string(string, encoding='ascii'): - """Given a string object, regardless of type, returns a representation of - that string in the native string type, encoding and decoding where - necessary. This assumes ASCII unless told otherwise. - """ - if isinstance(string, builtin_str): - out = string - else: - if is_py2: - out = string.encode(encoding) - else: - out = string.decode(encoding) - - return out - - # Moved outside of function to avoid recompile every call _CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$') _CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$') From 3ac70defff233359fbd715ad6e02350c2519291d Mon Sep 17 00:00:00 2001 From: Brian Bamsch Date: Tue, 27 Sep 2016 20:32:15 -0700 Subject: [PATCH 6/7] Update Comments & Magic Strings --- requests/_internal_utils.py | 10 ++++++++++ requests/utils.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/requests/_internal_utils.py b/requests/_internal_utils.py index 0456017c..87e2ca00 100644 --- a/requests/_internal_utils.py +++ b/requests/_internal_utils.py @@ -1,3 +1,13 @@ +# -*- coding: utf-8 -*- + +""" +requests._internal_utils +~~~~~~~~~~~~~~ + +Provides utility functions that are consumed internally by Requests +which depend on extremely few external helpers (such as compat) +""" + from .compat import is_py2, builtin_str diff --git a/requests/utils.py b/requests/utils.py index f7d4e37f..3b43ad7d 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -20,7 +20,7 @@ import warnings from . import __version__ from . import certs -# noinspection PyUnresolvedReferences +# to_native_string is unused here, but imported here for backwards compatibility from ._internal_utils import to_native_string from .compat import parse_http_list as _parse_list_header from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, From 087aeacee570e3e3adc6103015d4f8b056705850 Mon Sep 17 00:00:00 2001 From: Brian Bamsch Date: Tue, 27 Sep 2016 20:36:42 -0700 Subject: [PATCH 7/7] Change module of internal references to to_native_str() --- requests/auth.py | 3 ++- requests/models.py | 4 ++-- requests/sessions.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/requests/auth.py b/requests/auth.py index 49bcb24a..d1196daa 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -17,7 +17,8 @@ from base64 import b64encode from .compat import urlparse, str from .cookies import extract_cookies_to_jar -from .utils import parse_dict_header, to_native_string +from ._internal_utils import to_native_string +from .utils import parse_dict_header from .status_codes import codes CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' diff --git a/requests/models.py b/requests/models.py index ea25e356..57ab7700 100644 --- a/requests/models.py +++ b/requests/models.py @@ -29,11 +29,11 @@ from .packages.urllib3.exceptions import ( from .exceptions import ( HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, ContentDecodingError, ConnectionError, StreamConsumedError) +from ._internal_utils import to_native_string from .utils import ( guess_filename, get_auth_from_url, requote_uri, stream_decode_response_unicode, to_key_val_list, parse_header_links, - iter_slices, guess_json_utf, super_len, to_native_string, - check_header_validity) + iter_slices, guess_json_utf, super_len, check_header_validity) from .compat import ( cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO, is_py2, chardet, builtin_str, basestring) diff --git a/requests/sessions.py b/requests/sessions.py index b05150a4..971575b2 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -17,7 +17,8 @@ from .cookies import ( cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook -from .utils import to_key_val_list, default_headers, to_native_string +from ._internal_utils import to_native_string +from .utils import to_key_val_list, default_headers from .exceptions import ( TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) from .packages.urllib3._collections import RecentlyUsedContainer