From 15054aa3909da2383e8cfa9abcd296ba46a7200f Mon Sep 17 00:00:00 2001 From: Taylor Rose Date: Tue, 10 Oct 2017 13:21:11 -0400 Subject: [PATCH 01/38] Warn user about possible slowdown when using cryptography version < 1.3.4 --- AUTHORS.rst | 1 + HISTORY.rst | 2 ++ requests/__init__.py | 15 +++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index b87dc443..cdf8c516 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -178,3 +178,4 @@ Patches and Suggestions - Ryan Pineo (`@ryanpineo `_) - Ed Morley (`@edmorley `_) - Matt Liu (`@mlcrazy `_) +- Taylor Hoff (`@PrimordialHelios `_) diff --git a/HISTORY.rst b/HISTORY.rst index 7f18ea36..89a0b0dc 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,8 @@ dev **Improvements** +- Warn user about possible slowdown when using cryptography version < 1.3.4 + **Bugfixes** - Parsing empty ``Link`` headers with ``parse_header_links()`` no longer return one bogus entry diff --git a/requests/__init__.py b/requests/__init__.py index 268e7dcc..6fa855df 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -71,6 +71,17 @@ def check_compatibility(urllib3_version, chardet_version): assert patch >= 2 +def _check_cryptography(cryptography_version): + # cryptography < 1.3.4 + try: + cryptography_version = list(map(int, cryptography_version.split('.'))) + except ValueError: + return + + if cryptography_version < [1, 3, 4]: + warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warnings.warn(warning, RequestsDependencyWarning) + # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) @@ -83,6 +94,10 @@ except (AssertionError, ValueError): try: from urllib3.contrib import pyopenssl pyopenssl.inject_into_urllib3() + + # Check cryptography version + from cryptography import __version__ as cryptography_version + _check_cryptography(cryptography_version) except ImportError: pass From 4316af7ae82553f094b101d69b3e9c28ebbf0c18 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 15 Oct 2017 19:52:19 +0300 Subject: [PATCH 02/38] Python 3.7 not officially supported yet --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e0dc5e4f..5908162a 100644 --- a/README.rst +++ b/README.rst @@ -77,7 +77,7 @@ Requests is ready for today's web. - ``.netrc`` Support - Chunked Requests -Requests officially supports Python 2.6–2.7 & 3.3–3.7, and runs great on PyPy. +Requests officially supports Python 2.6–2.7 & 3.3–3.6, and runs great on PyPy. Installation ------------ From ec5804c706fcf2260ff85132b878851ffbbc2a6f Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 15 Oct 2017 20:04:51 +0300 Subject: [PATCH 03/38] Python 3.3 has already been dropped (#4231) --- README.rst | 2 +- requests/sessions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 5908162a..915f2a11 100644 --- a/README.rst +++ b/README.rst @@ -77,7 +77,7 @@ Requests is ready for today's web. - ``.netrc`` Support - Chunked Requests -Requests officially supports Python 2.6–2.7 & 3.3–3.6, and runs great on PyPy. +Requests officially supports Python 2.6–2.7 & 3.4–3.6, and runs great on PyPy. Installation ------------ diff --git a/requests/sessions.py b/requests/sessions.py index 6570e733..391f857d 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -39,7 +39,7 @@ from .models import REDIRECT_STATI # Preferred clock, based on which one is more accurate on a given system. if platform.system() == 'Windows': - try: # Python 3.3+ + try: # Python 3.4+ preferred_clock = time.perf_counter except AttributeError: # Earlier than Python 3. preferred_clock = time.clock From c14bd018adc715db2810c844258b392870fbc6fe Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 15 Oct 2017 19:48:10 -0700 Subject: [PATCH 04/38] Add missing trove classifier to document Python2 support --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7aa8d611..e3bc8761 100755 --- a/setup.py +++ b/setup.py @@ -81,6 +81,7 @@ setup( 'Natural Language :: English', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', @@ -98,4 +99,3 @@ setup( 'socks:sys_platform == "win32" and (python_version == "2.7" or python_version == "2.6")': ['win_inet_pton'], }, ) - From d3f14af44d9460a17c79de19ebdb65b3eec2e534 Mon Sep 17 00:00:00 2001 From: Alvaro Gutierrez Perez Date: Wed, 18 Oct 2017 19:27:06 +0200 Subject: [PATCH 05/38] Fix case-insensitive comparison in get_adapter() While trying to get the prefix for an url, the url was lowered before comparing but the prefix not, so if it contains non-lowercase characters (eg. https://api.example.com/sOmE_WeiRD_pReFIX/), it won't match. --- requests/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/sessions.py b/requests/sessions.py index 391f857d..bb57f5d8 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -696,7 +696,7 @@ class Session(SessionRedirectMixin): """ for (prefix, adapter) in self.adapters.items(): - if url.lower().startswith(prefix): + if url.lower().startswith(prefix.lower()): return adapter # Nothing matches :-/ From af88af64e6c5bc026247f18e2e6de20605f51e7f Mon Sep 17 00:00:00 2001 From: Alvaro Gutierrez Perez Date: Thu, 19 Oct 2017 16:36:17 +0200 Subject: [PATCH 06/38] Add test for Session.get_adapter() case-insensitivity --- tests/test_requests.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_requests.py b/tests/test_requests.py index 4d399518..76a528c9 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1351,6 +1351,18 @@ class TestRequests: assert 'http://' in s2.adapters assert 'https://' in s2.adapters + def test_session_get_adapter_prefix_matching_is_case_insensitive(self, httpbin): + mixed_case_prefix = 'hTtPs://eXamPle.CoM/MixEd_CAse_PREfix' + url_matching_prefix = mixed_case_prefix + '/full_url' + url_matching_prefix_with_different_case = 'HtTpS://exaMPLe.cOm/MiXeD_caSE_preFIX/another_url' + + s = requests.Session() + my_adapter = HTTPAdapter() + s.mount(mixed_case_prefix, my_adapter) + + assert s.get_adapter(url_matching_prefix) is my_adapter + assert s.get_adapter(url_matching_prefix_with_different_case) is my_adapter + def test_header_remove_is_case_insensitive(self, httpbin): # From issue #1321 s = requests.Session() From e11989e8ec5885ea36b9be25641fbd4ce9f0c4dd Mon Sep 17 00:00:00 2001 From: Alvaro Gutierrez Perez Date: Thu, 19 Oct 2017 16:37:09 +0200 Subject: [PATCH 07/38] Add test for Session.get_adapter() prefix matching --- tests/test_requests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_requests.py b/tests/test_requests.py index 76a528c9..a14a2c50 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1351,6 +1351,24 @@ class TestRequests: assert 'http://' in s2.adapters assert 'https://' in s2.adapters + def test_session_get_adapter_prefix_matching(self, httpbin): + prefix = 'https://example.com' + more_specific_prefix = prefix + '/some/path' + + url_matching_only_prefix = prefix + '/another/path' + url_matching_more_specific_prefix = more_specific_prefix + '/longer/path' + url_not_matching_prefix = 'https://another.example.com/' + + s = requests.Session() + prefix_adapter = HTTPAdapter() + more_specific_prefix_adapter = HTTPAdapter() + s.mount(prefix, prefix_adapter) + s.mount(more_specific_prefix, more_specific_prefix_adapter) + + assert s.get_adapter(url_matching_only_prefix) is prefix_adapter + assert s.get_adapter(url_matching_more_specific_prefix) is more_specific_prefix_adapter + assert s.get_adapter(url_not_matching_prefix) not in (prefix_adapter, more_specific_prefix_adapter) + def test_session_get_adapter_prefix_matching_is_case_insensitive(self, httpbin): mixed_case_prefix = 'hTtPs://eXamPle.CoM/MixEd_CAse_PREfix' url_matching_prefix = mixed_case_prefix + '/full_url' From d165b18b6e0c1aaabd8f652f88308008769f6625 Mon Sep 17 00:00:00 2001 From: Alvaro Gutierrez Perez Date: Thu, 19 Oct 2017 17:04:48 +0200 Subject: [PATCH 08/38] Split test in two better-defined tests --- tests/test_requests.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_requests.py b/tests/test_requests.py index a14a2c50..05fe63e3 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1369,16 +1369,24 @@ class TestRequests: assert s.get_adapter(url_matching_more_specific_prefix) is more_specific_prefix_adapter assert s.get_adapter(url_not_matching_prefix) not in (prefix_adapter, more_specific_prefix_adapter) - def test_session_get_adapter_prefix_matching_is_case_insensitive(self, httpbin): + def test_session_get_adapter_prefix_matching_mixed_case(self, httpbin): mixed_case_prefix = 'hTtPs://eXamPle.CoM/MixEd_CAse_PREfix' url_matching_prefix = mixed_case_prefix + '/full_url' - url_matching_prefix_with_different_case = 'HtTpS://exaMPLe.cOm/MiXeD_caSE_preFIX/another_url' s = requests.Session() my_adapter = HTTPAdapter() s.mount(mixed_case_prefix, my_adapter) assert s.get_adapter(url_matching_prefix) is my_adapter + + def test_session_get_adapter_prefix_matching_is_case_insensitive(self, httpbin): + mixed_case_prefix = 'hTtPs://eXamPle.CoM/MixEd_CAse_PREfix' + url_matching_prefix_with_different_case = 'HtTpS://exaMPLe.cOm/MiXeD_caSE_preFIX/another_url' + + s = requests.Session() + my_adapter = HTTPAdapter() + s.mount(mixed_case_prefix, my_adapter) + assert s.get_adapter(url_matching_prefix_with_different_case) is my_adapter def test_header_remove_is_case_insensitive(self, httpbin): From a05aac7007dd802401d59b6380ef95095836ed9b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 1 Oct 2017 01:08:14 +0900 Subject: [PATCH 09/38] avoid import platform platform module is relatively large: it takes about 5ms to import --- requests/sessions.py | 4 ++-- requests/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 391f857d..631fadb9 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -8,7 +8,7 @@ This module provides a Session object to manage and persist settings across requests (cookies, auth, proxies). """ import os -import platform +import sys import time from collections import Mapping from datetime import timedelta @@ -38,7 +38,7 @@ from .status_codes import codes from .models import REDIRECT_STATI # Preferred clock, based on which one is more accurate on a given system. -if platform.system() == 'Windows': +if sys.platform == 'win32': try: # Python 3.4+ preferred_clock = time.perf_counter except AttributeError: # Earlier than Python 3. diff --git a/requests/utils.py b/requests/utils.py index c52ce2d0..35fff043 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -14,10 +14,10 @@ import collections import contextlib import io import os -import platform import re import socket import struct +import sys import warnings from .__version__ import __version__ @@ -39,7 +39,7 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() -if platform.system() == 'Windows': +if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups def proxy_bypass_registry(host): From 1595e43812bdfcc4ef16567780e872b9eb79d9d2 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 15 Oct 2017 19:45:19 -0700 Subject: [PATCH 10/38] Include license file in the generated wheel package The wheel package format supports including the license file. This is done using the [metadata] section in the setup.cfg file. For additional information on this feature, see: https://wheel.readthedocs.io/en/stable/index.html#including-the-license-in-the-generated-wheel-file --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 2a9acf13..ed8a958e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [bdist_wheel] universal = 1 + +[metadata] +license_file = LICENSE From 2c4849defeda74c191c35c4b1997547b1aae425e Mon Sep 17 00:00:00 2001 From: Alex Chan Date: Mon, 23 Oct 2017 08:30:35 +0100 Subject: [PATCH 11/38] Add something to the docs about hooks on Session() --- docs/user/advanced.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 613df205..443b43e9 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -458,6 +458,15 @@ Let's print some request method arguments at runtime:: http://httpbin.org +You can also assign hooks to a ``Session`` instance. The hook will then be +called on every request made to the session. For example:: + + >>> s = requests.Session() + >>> s.hooks = dict(response=print_url) + >>> s.get('http://httpbin.org') + http://httpbin.org + + .. _custom-auth: Custom Authentication From 87f3b0a5595d875515de15927e1d2803dbfda73c Mon Sep 17 00:00:00 2001 From: Alex Chan Date: Mon, 23 Oct 2017 13:25:31 +0100 Subject: [PATCH 12/38] Switch to using dict literals, it's 2017 --- docs/user/advanced.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 443b43e9..93f86baa 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -436,7 +436,7 @@ You can assign a hook function on a per-request basis by passing a ``{hook_name: callback_function}`` dictionary to the ``hooks`` request parameter:: - hooks=dict(response=print_url) + hooks={'response': print_url} That ``callback_function`` will receive a chunk of data as its first argument. @@ -454,7 +454,7 @@ anything, nothing else is effected. Let's print some request method arguments at runtime:: - >>> requests.get('http://httpbin.org', hooks=dict(response=print_url)) + >>> requests.get('http://httpbin.org', hooks={'response': print_url}) http://httpbin.org @@ -462,7 +462,7 @@ You can also assign hooks to a ``Session`` instance. The hook will then be called on every request made to the session. For example:: >>> s = requests.Session() - >>> s.hooks = dict(response=print_url) + >>> s.hooks['response'].append(print_url) >>> s.get('http://httpbin.org') http://httpbin.org From 40c5a8b0c28e2756e83ecfab0160ca3be37391e1 Mon Sep 17 00:00:00 2001 From: Alex Chan Date: Mon, 23 Oct 2017 13:25:46 +0100 Subject: [PATCH 13/38] Clarify that a Session can have multiple hooks --- docs/user/advanced.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 93f86baa..c24f6e5b 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -458,8 +458,8 @@ Let's print some request method arguments at runtime:: http://httpbin.org -You can also assign hooks to a ``Session`` instance. The hook will then be -called on every request made to the session. For example:: +You can also add hooks to a ``Session`` instance. Any hooks you add will then +be called on every request made to the session. For example:: >>> s = requests.Session() >>> s.hooks['response'].append(print_url) @@ -467,6 +467,9 @@ called on every request made to the session. For example:: http://httpbin.org +A ``Session`` can have multiple hooks, which will be called in the order +they are added. + .. _custom-auth: Custom Authentication From c5ed41e00a2010fc251ed5655d9f1923c345878a Mon Sep 17 00:00:00 2001 From: Alex Chan Date: Tue, 24 Oct 2017 07:27:55 +0100 Subject: [PATCH 14/38] Add an example of two hooks --- docs/user/advanced.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index c24f6e5b..587b3fdc 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -452,12 +452,24 @@ If the callback function returns a value, it is assumed that it is to replace the data that was passed in. If the function doesn't return anything, nothing else is effected. +:: + + def record_hook(r, *args, **kwargs): + r.hook_called = True + return r + Let's print some request method arguments at runtime:: >>> requests.get('http://httpbin.org', hooks={'response': print_url}) http://httpbin.org +You can add multiple hooks to a single request. Let's call two hooks at once:: + + >>> r = requests.get('http://httpbin.org', hooks={'response': [print_url, record_hook]}) + >>> r.hook_called + True + You can also add hooks to a ``Session`` instance. Any hooks you add will then be called on every request made to the session. For example:: From c86b09b3c67a437f8851d3be9232b0a1c8585e20 Mon Sep 17 00:00:00 2001 From: Arthur Vigil Date: Sun, 5 Nov 2017 10:50:35 -0800 Subject: [PATCH 15/38] support extraction of certificate bundle from a zip archive --- AUTHORS.rst | 1 + HISTORY.rst | 3 +++ requests/adapters.py | 8 ++++---- requests/utils.py | 34 ++++++++++++++++++++++++++++++++++ tests/test_utils.py | 30 +++++++++++++++++++++++++++++- 5 files changed, 71 insertions(+), 5 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index cdf8c516..1bec7846 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -179,3 +179,4 @@ Patches and Suggestions - Ed Morley (`@edmorley `_) - Matt Liu (`@mlcrazy `_) - Taylor Hoff (`@PrimordialHelios `_) +- Arthur Vigil (`@ahvigil `_) diff --git a/HISTORY.rst b/HISTORY.rst index 89a0b0dc..e6281c1e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,9 @@ dev **Bugfixes** - Parsing empty ``Link`` headers with ``parse_header_links()`` no longer return one bogus entry +- Fixed issue where loading the default certificate bundle from a zip archive + would raise an ``IOError`` + 2.18.4 (2017-08-15) +++++++++++++++++++ diff --git a/requests/adapters.py b/requests/adapters.py index 00f8792b..cdaabdbe 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -28,9 +28,9 @@ from urllib3.exceptions import ResponseError from .models import Response from .compat import urlparse, basestring -from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, - prepend_scheme_if_needed, get_auth_from_url, urldefragauth, - select_proxy) +from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, + get_encoding_from_headers, prepend_scheme_if_needed, + get_auth_from_url, urldefragauth, select_proxy) from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, @@ -219,7 +219,7 @@ class HTTPAdapter(BaseAdapter): cert_loc = verify if not cert_loc: - cert_loc = DEFAULT_CA_BUNDLE_PATH + cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " diff --git a/requests/utils.py b/requests/utils.py index 35fff043..1cba5a93 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -18,7 +18,9 @@ import re import socket import struct import sys +import tempfile import warnings +import zipfile from .__version__ import __version__ from . import certs @@ -216,6 +218,38 @@ def guess_filename(obj): return os.path.basename(name) +def extract_zipped_paths(path): + """Replace nonexistant paths that look like they refer to a member of a zip + archive with the location of an extracted copy of the target, or else + just return the provided path unchanged. + """ + if os.path.exists(path): + # this is already a valid path, no need to do anything further + return path + + # find the first valid part of the provided path and treat that as a zip archive + # assume the rest of the path is the name of a member in the archive + archive, member = os.path.split(path) + while archive and not os.path.exists(archive): + archive, prefix = os.path.split(archive) + member = '/'.join([prefix, member]) + + if not zipfile.is_zipfile(archive): + return path + + zip_file = zipfile.ZipFile(archive) + if member not in zip_file.namelist(): + return path + + # we have a valid zip archive and a valid member of that archive + tmp = tempfile.gettempdir() + extracted_path = os.path.join(tmp, *member.split('/')) + if not os.path.exists(extracted_path): + extracted_path = zip_file.extract(member, path=tmp) + + return extracted_path + + def from_key_val_list(value): """Take an object and test to see if it can be represented as a dictionary. Unless it can not be represented as such, return an diff --git a/tests/test_utils.py b/tests/test_utils.py index 32e4d4a5..2292a8f0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,14 +2,16 @@ import os import copy +import filecmp from io import BytesIO +import zipfile import pytest from requests import compat from requests.cookies import RequestsCookieJar from requests.structures import CaseInsensitiveDict from requests.utils import ( - address_in_network, dotted_netmask, + address_in_network, dotted_netmask, extract_zipped_paths, get_auth_from_url, get_encoding_from_headers, get_encodings_from_content, get_environ_proxies, guess_filename, guess_json_utf, is_ipv4_address, @@ -256,6 +258,32 @@ class TestGuessFilename: assert isinstance(result, expected_type) +class TestExtractZippedPaths: + + @pytest.mark.parametrize( + 'path', ( + '/', + __file__, + pytest.__file__, + '/etc/invalid/location', + )) + def test_unzipped_paths_unchanged(self, path): + assert path == extract_zipped_paths(path) + + def test_zipped_paths_extracted(self, tmpdir): + zipped_py = tmpdir.join('test.zip') + with zipfile.ZipFile(zipped_py.strpath, 'w') as f: + f.write(__file__) + + _, name = os.path.splitdrive(__file__) + zipped_path = os.path.join(zipped_py.strpath, name.lstrip(r'\/')) + extracted_path = extract_zipped_paths(zipped_path) + + assert extracted_path != zipped_path + assert os.path.exists(extracted_path) + assert filecmp.cmp(extracted_path, __file__) + + class TestContentEncodingDetection: def test_none(self): From 9a8a826f226e6973d72914b4a4fc806e0b5036f4 Mon Sep 17 00:00:00 2001 From: Nehal J Wani Date: Thu, 26 Oct 2017 10:33:05 -0400 Subject: [PATCH 16/38] Check if host is invalid for proxy According to RFC3986, the authority section can be empty for a given URL, however, for a proxy URL, it shouldn't be. This patch adds a check to verify that the parsed URL will have a valid host before creating the proxy manager. Fixes #4353 --- AUTHORS.rst | 1 + HISTORY.rst | 1 + requests/adapters.py | 7 ++++++- requests/exceptions.py | 4 ++++ tests/test_requests.py | 15 ++++++++++++++- 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 1bec7846..8379f65c 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -180,3 +180,4 @@ Patches and Suggestions - Matt Liu (`@mlcrazy `_) - Taylor Hoff (`@PrimordialHelios `_) - Arthur Vigil (`@ahvigil `_) +- Nehal J Wani (`@nehaljwani `_) diff --git a/HISTORY.rst b/HISTORY.rst index e6281c1e..79202df6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ dev **Improvements** - Warn user about possible slowdown when using cryptography version < 1.3.4 +- Check for invalid host in proxy URL, before forwarding request to adapter. **Bugfixes** diff --git a/requests/adapters.py b/requests/adapters.py index cdaabdbe..bc01e336 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -13,6 +13,7 @@ import socket from urllib3.poolmanager import PoolManager, proxy_from_url from urllib3.response import HTTPResponse +from urllib3.util import parse_url from urllib3.util import Timeout as TimeoutSauce from urllib3.util.retry import Retry from urllib3.exceptions import ClosedPoolError @@ -34,7 +35,7 @@ from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL) from .auth import _basic_auth_str try: @@ -300,6 +301,10 @@ class HTTPAdapter(BaseAdapter): if proxy: proxy = prepend_scheme_if_needed(proxy, 'http') + proxy_url = parse_url(proxy) + if not proxy_url.host: + raise InvalidProxyURL("Please check proxy URL. It is malformed" + " and could be missing the host.") proxy_manager = self.proxy_manager_for(proxy) conn = proxy_manager.connection_from_url(url) else: diff --git a/requests/exceptions.py b/requests/exceptions.py index be7eaed6..a80cad80 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -85,6 +85,10 @@ class InvalidHeader(RequestException, ValueError): """The header value provided was somehow invalid.""" +class InvalidProxyURL(InvalidURL): + """The proxy URL provided is invalid.""" + + class ChunkedEncodingError(RequestException): """The server declared chunked encoding but sent an invalid chunk.""" diff --git a/tests/test_requests.py b/tests/test_requests.py index 05fe63e3..e6a026f2 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -23,7 +23,7 @@ from requests.cookies import ( from requests.exceptions import ( ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL, MissingSchema, ReadTimeout, Timeout, RetryError, TooManyRedirects, - ProxyError, InvalidHeader, UnrewindableBodyError, SSLError) + ProxyError, InvalidHeader, UnrewindableBodyError, SSLError, InvalidProxyURL) from requests.models import PreparedRequest from requests.structures import CaseInsensitiveDict from requests.sessions import SessionRedirectMixin @@ -526,6 +526,19 @@ class TestRequests: with pytest.raises(ProxyError): requests.get('http://localhost:1', proxies={'http': 'non-resolvable-address'}) + def test_proxy_error_on_bad_url(self, httpbin, httpbin_secure): + with pytest.raises(InvalidProxyURL): + requests.get(httpbin_secure(), proxies={'https': 'http:/badproxyurl:3128'}) + + with pytest.raises(InvalidProxyURL): + requests.get(httpbin(), proxies={'http': 'http://:8080'}) + + with pytest.raises(InvalidProxyURL): + requests.get(httpbin_secure(), proxies={'https': 'https://'}) + + with pytest.raises(InvalidProxyURL): + requests.get(httpbin(), proxies={'http': 'http:///example.com:8080'}) + def test_basicauth_with_netrc(self, httpbin): auth = ('user', 'pass') wrong_auth = ('wronguser', 'wrongpass') From 7f08ad3b6c633193c80cf26eb2dc895ea41ed2ae Mon Sep 17 00:00:00 2001 From: Tal Einat Date: Thu, 16 Nov 2017 09:59:00 +0200 Subject: [PATCH 17/38] Corrent HTTP -> HTML in quickstart doc --- docs/user/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 7fe1e0d2..b5288ea3 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -110,7 +110,7 @@ using, and change it, using the ``r.encoding`` property:: If you change the encoding, Requests will use the new value of ``r.encoding`` whenever you call ``r.text``. You might want to do this in any situation where you can apply special logic to work out what the encoding of the content will -be. For example, HTTP and XML have the ability to specify their encoding in +be. For example, HTML and XML have the ability to specify their encoding in their body. In situations like this, you should use ``r.content`` to find the encoding, and then set ``r.encoding``. This will let you use ``r.text`` with the correct encoding. From 7cc3d8dc6a24975617e388e674d93251c74749f1 Mon Sep 17 00:00:00 2001 From: Dave Shawley Date: Fri, 17 Nov 2017 16:37:08 -0500 Subject: [PATCH 18/38] docs/quickstart: clarify raw response reading. --- docs/user/quickstart.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index b5288ea3..6829d592 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -189,6 +189,14 @@ download, the above is the preferred and recommended way to retrieve the content. Note that ``chunk_size`` can be freely adjusted to a number that may better fit your use cases. +.. note:: + + An important note about using ``Response.iter_content`` versus ``Response.raw``. + ``Response.iter_content`` will automatically decode the ``gzip`` and ``deflate`` + transfer-encodings. ``Response.raw`` is a raw stream of bytes -- it does not + transform the response content. If you really need access to the bytes as they + were returned, use ``Response.raw``. + Custom Headers -------------- From 775cde0914265d7dab2bc2501ed7abe6b85c4bae Mon Sep 17 00:00:00 2001 From: Alex Chan Date: Mon, 20 Nov 2017 09:16:35 +0000 Subject: [PATCH 19/38] Clarify that Response.ok will *only* return True/False --- requests/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 4041cac3..ce4e284e 100644 --- a/requests/models.py +++ b/requests/models.py @@ -686,11 +686,11 @@ class Response(object): @property def ok(self): - """Returns True if :attr:`status_code` is less than 400. + """Returns True if :attr:`status_code` is less than 400, False if not. This attribute checks if the status code of the response is between 400 and 600 to see if there was a client error or a server error. If - the status code, is between 200 and 400, this will return True. This + the status code is between 200 and 400, this will return True. This is **not** a check to see if the response code is ``200 OK``. """ try: From acd2645444d280ac0c5d2d227fdd222cb1ac609c Mon Sep 17 00:00:00 2001 From: Mingyuan Xia Date: Tue, 21 Nov 2017 04:01:04 +0800 Subject: [PATCH 20/38] #4373, fix possible winreg value type difference (#4377) * #4373, fix possible winreg value type difference * add a test for ProxyOverride and ProxyEnable on win32 * add tests for winreg key ProxyEnable with two possible types * fixing AppVeyor failures --- requests/utils.py | 6 ++++-- tests/test_utils.py | 7 ++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 1cba5a93..42daa2d7 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -52,8 +52,10 @@ if sys.platform == 'win32': try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') - proxyEnable = winreg.QueryValueEx(internetSettings, - 'ProxyEnable')[0] + # ProxyEnable could be REG_SZ or REG_DWORD, normalizing it + proxyEnable = int(winreg.QueryValueEx(internetSettings, + 'ProxyEnable')[0]) + # ProxyOverride is almost always a string proxyOverride = winreg.QueryValueEx(internetSettings, 'ProxyOverride')[0] except OSError: diff --git a/tests/test_utils.py b/tests/test_utils.py index 2292a8f0..2dd16923 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,6 +5,7 @@ import copy import filecmp from io import BytesIO import zipfile +from collections import deque import pytest from requests import compat @@ -666,6 +667,7 @@ def test_should_bypass_proxies_win_registry(url, expected, override, pass ie_settings = RegHandle() + proxyEnableValues = deque([1, "1"]) def OpenKey(key, subkey): return ie_settings @@ -673,7 +675,9 @@ def test_should_bypass_proxies_win_registry(url, expected, override, def QueryValueEx(key, value_name): if key is ie_settings: if value_name == 'ProxyEnable': - return [1] + # this could be a string (REG_SZ) or a 32-bit number (REG_DWORD) + proxyEnableValues.rotate() + return [proxyEnableValues[0]] elif value_name == 'ProxyOverride': return [override] @@ -684,6 +688,7 @@ def test_should_bypass_proxies_win_registry(url, expected, override, monkeypatch.setenv('NO_PROXY', '') monkeypatch.setattr(winreg, 'OpenKey', OpenKey) monkeypatch.setattr(winreg, 'QueryValueEx', QueryValueEx) + assert should_bypass_proxies(url, None) == expected @pytest.mark.parametrize( From 39446def395672b96642ad9b317fab09ee771be1 Mon Sep 17 00:00:00 2001 From: Daniel Roseman Date: Fri, 3 Nov 2017 13:53:08 +0000 Subject: [PATCH 21/38] Clarify behaviour of `json` parameter. `json` is ignored if `data` or `files` is not empty. --- docs/user/quickstart.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 6829d592..1a2c6fbf 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -281,6 +281,7 @@ the ``json`` parameter (added in version 2.4.2) and it will be encoded automatic >>> r = requests.post(url, json=payload) +Note, the ``json`` parameter is ignored if either ``data`` or ``files`` is passed. POST a Multipart-Encoded File ----------------------------- From 19919b44c4af95f125704c902acecdf83d70a3e4 Mon Sep 17 00:00:00 2001 From: Zane Bitter Date: Tue, 21 Nov 2017 12:39:22 -0500 Subject: [PATCH 22/38] Add documentation for available status codes There was no way to determine what actual names were available outside of looking at the source code. They were not listed in the documentation or accessible through the interactive help. In addition, doing `pydoc requests.status_codes` displayed some pretty unhelpful information - the utf-8 encoding string was included in the module name, there was no description, and internal variables used for initialisation leaked into the module scope: DATA code = 511 codes = title = 'network_authentication' titles = ('network_authentication_required', 'network_auth', ... This change prevents the internal variables from leaking, adds a docstring (which has the side-effect of correcting the module name), and appends information on the allowed status code names to the docstring when the module is initialised. The improved module documentation is then used in the API documentation to provide another easy reference to the complete list of status codes. --- docs/api.rst | 12 +----------- requests/status_codes.py | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index ed61bb38..c3e00e54 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -109,17 +109,7 @@ Status Code Lookup .. autoclass:: requests.codes -:: - - >>> requests.codes['temporary_redirect'] - 307 - - >>> requests.codes.teapot - 418 - - >>> requests.codes['\o/'] - 200 - +.. automodule:: requests.status_codes Migrating to 1.x diff --git a/requests/status_codes.py b/requests/status_codes.py index dee89190..96b86ddb 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -1,5 +1,22 @@ # -*- coding: utf-8 -*- +""" +The ``codes`` object defines a mapping from common names for HTTP statuses +to their numerical codes, accessible either as attributes or as dictionary +items. + +>>> requests.codes['temporary_redirect'] +307 +>>> requests.codes.teapot +418 +>>> requests.codes['\o/'] +200 + +Some codes have multiple names, and both upper- and lower-case versions of +the names are allowed. For example, ``codes.ok``, ``codes.OK``, and +``codes.okay`` all correspond to the HTTP status code 200. +""" + from .structures import LookupDict _codes = { @@ -84,8 +101,19 @@ _codes = { codes = LookupDict(name='status_codes') -for code, titles in _codes.items(): - for title in titles: - setattr(codes, title, code) - if not title.startswith(('\\', '/')): - setattr(codes, title.upper(), code) +def _init(): + for code, titles in _codes.items(): + for title in titles: + setattr(codes, title, code) + if not title.startswith(('\\', '/')): + setattr(codes, title.upper(), code) + + def doc(code): + names = ', '.join('``%s``' % n for n in _codes[code]) + return '* %d: %s' % (code, names) + + global __doc__ + __doc__ = (__doc__ + '\n' + + '\n'.join(doc(code) for code in sorted(_codes))) + +_init() From 714c9dc96759942226a0dc16fa1e3d6d51f56291 Mon Sep 17 00:00:00 2001 From: Anton Fedchin Date: Fri, 10 Nov 2017 10:32:41 +0300 Subject: [PATCH 23/38] utils: winreg module may not exist like on windows universal platform. --- requests/utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 42daa2d7..f9565287 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -45,10 +45,14 @@ if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups def proxy_bypass_registry(host): - if is_py3: - import winreg - else: - import _winreg as winreg + try: + if is_py3: + import winreg + else: + import _winreg as winreg + except ImportError: + return False + try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') From 351ec982bb38a069a6b236b8d77dc09f26c2f444 Mon Sep 17 00:00:00 2001 From: Anton Fedchin Date: Sat, 25 Nov 2017 12:07:41 +0300 Subject: [PATCH 24/38] update changelog --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 79202df6..b099ecdb 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,7 @@ dev - Parsing empty ``Link`` headers with ``parse_header_links()`` no longer return one bogus entry - Fixed issue where loading the default certificate bundle from a zip archive would raise an ``IOError`` +- Fixed issue with unexpected ``ImportError`` on windows system which do not support ``winreg`` module 2.18.4 (2017-08-15) From d8666e190631b5330c2851bd354d07831afba114 Mon Sep 17 00:00:00 2001 From: dbairaktaris1 Date: Sun, 31 Dec 2017 14:46:15 -0600 Subject: [PATCH 25/38] Reduce overall memory usage of Requests module by removing cgi module dependency in utils.py. Instead wrote a nested function to parse header and return content type and params. --- requests/utils.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index f9565287..a1a3a7cb 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -8,7 +8,6 @@ This module provides utility functions that are used within Requests that are also useful for external consumption. """ -import cgi import codecs import collections import contextlib @@ -453,13 +452,28 @@ def get_encoding_from_headers(headers): :param headers: dictionary to extract encoding from. :rtype: str """ + def parse_header(content_type): + #Inner function to parse header + content_type_and_params_delimiter = ';' + + #append delimiter on end to ensure atleast two elements when split by ';' + content_type += content_type_and_params_delimiter + + tokens = content_type.split(content_type_and_params_delimiter) + content_type_index = 0 + params_index = 1 + + content_type = tokens[content_type_index] + params = tokens[params_index] + params_dict = dict(param.split('=') for param in params.split()) + return content_type,params_dict content_type = headers.get('content-type') if not content_type: return None - content_type, params = cgi.parse_header(content_type) + content_type, params = parse_header(content_type) if 'charset' in params: return params['charset'].strip("'\"") From cef08304197b8b8747015d94a1700716202355ee Mon Sep 17 00:00:00 2001 From: dbairaktaris1 Date: Sun, 31 Dec 2017 15:02:39 -0600 Subject: [PATCH 26/38] clean --- requests/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index a1a3a7cb..37e3e27d 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -454,9 +454,8 @@ def get_encoding_from_headers(headers): """ def parse_header(content_type): #Inner function to parse header - content_type_and_params_delimiter = ';' - #append delimiter on end to ensure atleast two elements when split by ';' + content_type_and_params_delimiter = ';' content_type += content_type_and_params_delimiter tokens = content_type.split(content_type_and_params_delimiter) From 19cfec28a8ee8f2044a883bc25406f7865fffeac Mon Sep 17 00:00:00 2001 From: dbairaktaris1 Date: Sun, 31 Dec 2017 22:18:19 -0600 Subject: [PATCH 27/38] CI --- .idea/misc.xml | 4 + .idea/modules.xml | 8 + .idea/requests_open_source.iml | 11 + .idea/vcs.xml | 6 + .idea/workspace.xml | 398 +++++++++++++++++++++++++++++++++ .travis.yml | 4 +- 6 files changed, 429 insertions(+), 2 deletions(-) create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/requests_open_source.iml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..96eb542b --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..85d7cf7a --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/requests_open_source.iml b/.idea/requests_open_source.iml new file mode 100644 index 00000000..67116063 --- /dev/null +++ b/.idea/requests_open_source.iml @@ -0,0 +1,11 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..94a25f7f --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 00000000..f41d18e5 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,398 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + header + encoding + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1514747970257 - - - 1514753175234 - - - 1514754159918 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 1ee87011..b7693f63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,8 @@ language: python python: # - "2.6" - "2.7" -# - "3.4" -# - "3.5" + - "3.4" + - "3.5" - "3.6" # - "3.7-dev" # - "pypy" -- appears to hang From 1988d9cf72c3a3a6da87968e04ad57fb32df01cb Mon Sep 17 00:00:00 2001 From: dbairaktaris1 Date: Mon, 1 Jan 2018 14:20:55 -0600 Subject: [PATCH 29/38] Move nested function up to module level and rename. Add more tests for function. --- requests/utils.py | 46 ++++++++++++++++++++++++++++++--------------- tests/test_utils.py | 37 +++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 37e3e27d..118e7e1b 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -446,33 +446,49 @@ def get_encodings_from_content(content): xml_re.findall(content)) +def _parse_content_type_header(header): + """Returns content type and parameters from given header + + :param header: string + :return: tuple containing content type and dictionary of + parameters + """ + if not header: + return None + # append delimiter on end to ensure at least two elements when split by ';' + header += ';' + # split content type's main value from params + tokens = header.split(';', 1) + content_type_index = 0 + params_index = 1 + + content_type = tokens[content_type_index].strip() + params = tokens[params_index] + params_dict = dict() + + for param in params.split(';'): + if param and not param.isspace(): + param = param.strip() + key, value = param, True + if '=' in param: + param_tokens = [x.strip('\'" ') for x in param.split('=', 1)] + key, value = param_tokens[0], param_tokens[1] + params_dict[key] = value + return content_type, params_dict + def get_encoding_from_headers(headers): """Returns encodings from given HTTP Header Dict. :param headers: dictionary to extract encoding from. :rtype: str """ - def parse_header(content_type): - #Inner function to parse header - #append delimiter on end to ensure atleast two elements when split by ';' - content_type_and_params_delimiter = ';' - content_type += content_type_and_params_delimiter - - tokens = content_type.split(content_type_and_params_delimiter) - content_type_index = 0 - params_index = 1 - - content_type = tokens[content_type_index] - params = tokens[params_index] - params_dict = dict(param.split('=') for param in params.split()) - return content_type,params_dict content_type = headers.get('content-type') if not content_type: return None - content_type, params = parse_header(content_type) + content_type, params = _parse_content_type_header(content_type) if 'charset' in params: return params['charset'].strip("'\"") diff --git a/tests/test_utils.py b/tests/test_utils.py index 2dd16923..e734b8f8 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -13,7 +13,7 @@ from requests.cookies import RequestsCookieJar from requests.structures import CaseInsensitiveDict from requests.utils import ( address_in_network, dotted_netmask, extract_zipped_paths, - get_auth_from_url, get_encoding_from_headers, + get_auth_from_url, _parse_content_type_header, get_encoding_from_headers, get_encodings_from_content, get_environ_proxies, guess_filename, guess_json_utf, is_ipv4_address, is_valid_cidr, iter_slices, parse_dict_header, @@ -470,6 +470,41 @@ def test_parse_dict_header(value, expected): assert parse_dict_header(value) == expected +@pytest.mark.parametrize( + 'value, expected', ( + ( + None, + None + ), +( + '', + None + ), + ( + 'application/xml', + ('application/xml', dict()) + ), + ( + 'application/json ; charset=utf-8', + ('application/json', {'charset': 'utf-8'}) + ), + ( + 'text/plain', + ('text/plain', dict()) + ), + ( + 'multipart/form-data; boundary = something ; \'boundary2=something_else\' ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), + ( + 'application/json ;; ; ', + ('application/json', dict()) + ) + )) +def test__parse_content_type_header(value, expected): + assert _parse_content_type_header(value) == expected + + @pytest.mark.parametrize( 'value, expected', ( ( From 071796d83f1bfb79793170945fdb4f623a1f344a Mon Sep 17 00:00:00 2001 From: dbairaktaris1 Date: Wed, 3 Jan 2018 23:40:08 -0600 Subject: [PATCH 30/38] implement changes after code review --- requests/utils.py | 17 +++++------------ tests/test_utils.py | 8 -------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 118e7e1b..44b3e016 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -453,20 +453,12 @@ def _parse_content_type_header(header): :return: tuple containing content type and dictionary of parameters """ - if not header: - return None - # append delimiter on end to ensure at least two elements when split by ';' - header += ';' - # split content type's main value from params - tokens = header.split(';', 1) - content_type_index = 0 - params_index = 1 - content_type = tokens[content_type_index].strip() - params = tokens[params_index] - params_dict = dict() + tokens = header.split(';') + content_type, params = tokens[0].strip(), tokens[1:] + params_dict = {} # Using dict is actually slower than a dictionary literal. Weird but tru - for param in params.split(';'): + for param in params: if param and not param.isspace(): param = param.strip() key, value = param, True @@ -476,6 +468,7 @@ def _parse_content_type_header(header): params_dict[key] = value return content_type, params_dict + def get_encoding_from_headers(headers): """Returns encodings from given HTTP Header Dict. diff --git a/tests/test_utils.py b/tests/test_utils.py index e734b8f8..f89d15aa 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -472,14 +472,6 @@ def test_parse_dict_header(value, expected): @pytest.mark.parametrize( 'value, expected', ( - ( - None, - None - ), -( - '', - None - ), ( 'application/xml', ('application/xml', dict()) From 80a790443e693d982296db93ceebc9135b6efb9c Mon Sep 17 00:00:00 2001 From: dbairaktaris1 Date: Wed, 3 Jan 2018 23:41:41 -0600 Subject: [PATCH 31/38] implement changes after code review --- requests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/utils.py b/requests/utils.py index 44b3e016..958f694d 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -456,7 +456,7 @@ def _parse_content_type_header(header): tokens = header.split(';') content_type, params = tokens[0].strip(), tokens[1:] - params_dict = {} # Using dict is actually slower than a dictionary literal. Weird but tru + params_dict = {} for param in params: if param and not param.isspace(): From cb0914407b6bb8153c8be5d52bc497e1d10b04ac Mon Sep 17 00:00:00 2001 From: dbairaktaris1 Date: Thu, 4 Jan 2018 10:30:50 -0600 Subject: [PATCH 32/38] Continue to refactor, remove list comprehension, add double quotes test case. --- AUTHORS.rst | 1 + requests/utils.py | 16 ++++++++++------ tests/test_utils.py | 12 ++++++++---- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 8379f65c..481ac6c7 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -181,3 +181,4 @@ Patches and Suggestions - Taylor Hoff (`@PrimordialHelios `_) - Arthur Vigil (`@ahvigil `_) - Nehal J Wani (`@nehaljwani `_) +- Demetrios Bairaktaris (`@DemetriosBairaktaris `_) diff --git a/requests/utils.py b/requests/utils.py index 958f694d..6c2bf5f5 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -456,15 +456,19 @@ def _parse_content_type_header(header): tokens = header.split(';') content_type, params = tokens[0].strip(), tokens[1:] - params_dict = {} + params_dict = {} + items_to_strip = "\"' " for param in params: - if param and not param.isspace(): - param = param.strip() + param = param.strip() + if param: key, value = param, True - if '=' in param: - param_tokens = [x.strip('\'" ') for x in param.split('=', 1)] - key, value = param_tokens[0], param_tokens[1] + index_of_equals = param.find("=") + if index_of_equals != -1: + before_equals = slice(0, index_of_equals) + after_equals = slice(index_of_equals + 1, len(param)) + key = param[before_equals].strip(items_to_strip) + value = param[after_equals].strip(items_to_strip) params_dict[key] = value return content_type, params_dict diff --git a/tests/test_utils.py b/tests/test_utils.py index f89d15aa..53d27a26 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -474,7 +474,7 @@ def test_parse_dict_header(value, expected): 'value, expected', ( ( 'application/xml', - ('application/xml', dict()) + ('application/xml', {}) ), ( 'application/json ; charset=utf-8', @@ -482,15 +482,19 @@ def test_parse_dict_header(value, expected): ), ( 'text/plain', - ('text/plain', dict()) + ('text/plain', {}) ), ( 'multipart/form-data; boundary = something ; \'boundary2=something_else\' ; no_equals ', ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) ), ( - 'application/json ;; ; ', - ('application/json', dict()) + 'multipart/form-data; boundary = something ; \"boundary2=something_else\" ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), + ( + 'application/json ; ; ', + ('application/json', {}) ) )) def test__parse_content_type_header(value, expected): From 7deee699ada6e5ec0d41c7561d9b5fa4cd80e535 Mon Sep 17 00:00:00 2001 From: dbairaktaris1 Date: Thu, 4 Jan 2018 10:48:17 -0600 Subject: [PATCH 33/38] slice function removed --- requests/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 6c2bf5f5..8c1b9bec 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -465,10 +465,8 @@ def _parse_content_type_header(header): key, value = param, True index_of_equals = param.find("=") if index_of_equals != -1: - before_equals = slice(0, index_of_equals) - after_equals = slice(index_of_equals + 1, len(param)) - key = param[before_equals].strip(items_to_strip) - value = param[after_equals].strip(items_to_strip) + key = param[:index_of_equals].strip(items_to_strip) + value = param[index_of_equals + 1:].strip(items_to_strip) params_dict[key] = value return content_type, params_dict From e0ab287317fcde8fa4631bc7bee5aa1749bc4ac5 Mon Sep 17 00:00:00 2001 From: dbairaktaris1 Date: Thu, 4 Jan 2018 10:59:47 -0600 Subject: [PATCH 34/38] added more to test scenarios --- tests/test_utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 53d27a26..01cabe23 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -484,12 +484,20 @@ def test_parse_dict_header(value, expected): 'text/plain', ('text/plain', {}) ), + ( + 'multipart/form-data; boundary = something ; boundary2=\'something_else\' ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), + ( + 'multipart/form-data; boundary = something ; boundary2="something_else" ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), ( 'multipart/form-data; boundary = something ; \'boundary2=something_else\' ; no_equals ', ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) ), ( - 'multipart/form-data; boundary = something ; \"boundary2=something_else\" ; no_equals ', + 'multipart/form-data; boundary = something ; "boundary2=something_else" ; no_equals ', ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) ), ( From 030dcce20cb8b4a82e7b0e5e09b7a8f33121938d Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sat, 6 Jan 2018 11:20:54 -0800 Subject: [PATCH 35/38] Prefer https over http for links in the documentation - Fixed Read the Docs links - Fixed GitHub links - Fixed PyPI links --- README.rst | 2 +- docs/_templates/sidebarintro.html | 10 +++++----- docs/_templates/sidebarlogo.html | 4 ++-- docs/community/out-there.rst | 2 +- docs/community/recommended.rst | 2 +- docs/conf.py | 2 +- docs/dev/todo.rst | 2 +- docs/user/advanced.rst | 6 +++--- docs/user/authentication.rst | 10 +++++----- 9 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index 915f2a11..c7b033be 100644 --- a/README.rst +++ b/README.rst @@ -105,6 +105,6 @@ How to Contribute #. Write a test which shows that the bug was fixed or that the feature works as expected. #. Send a pull request and bug the maintainer until it gets merged and published. :) Make sure to add yourself to AUTHORS_. -.. _`the repository`: http://github.com/requests/requests +.. _`the repository`: https://github.com/requests/requests .. _AUTHORS: https://github.com/requests/requests/blob/master/AUTHORS.rst .. _Contributor Friendly: https://github.com/requests/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index 5b437d85..5087dbc0 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -5,7 +5,7 @@

-

@@ -20,7 +20,7 @@

Stay Informed

Receive updates on new releases and upcoming projects.

-

@@ -48,9 +48,9 @@

-
  • Requests @ GitHub
    • -
  • Requests @ PyPI
    • -
  • Issue Tracker
    • +
  • Requests @ GitHub
    • +
  • Requests @ PyPI
    • +
  • Issue Tracker
  • Release History
    • diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index b31c3477..8a7d8d9a 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -4,7 +4,7 @@

    -

    @@ -25,7 +25,7 @@

    If you enjoy using this project, Say Thanks!

    -

    diff --git a/docs/community/out-there.rst b/docs/community/out-there.rst index 5ce5f79f..63e70169 100644 --- a/docs/community/out-there.rst +++ b/docs/community/out-there.rst @@ -18,7 +18,7 @@ Articles & Talks - `Python for the Web `_ teaches how to use Python to interact with the web, using Requests. - `Daniel Greenfeld's Review of Requests `_ - `My 'Python for Humans' talk `_ ( `audio `_ ) -- `Issac Kelly's 'Consuming Web APIs' talk `_ +- `Issac Kelly's 'Consuming Web APIs' talk `_ - `Blog post about Requests via Yum `_ - `Russian blog post introducing Requests `_ - `Sending JSON in Requests `_ diff --git a/docs/community/recommended.rst b/docs/community/recommended.rst index 0f652d54..88dcce8d 100644 --- a/docs/community/recommended.rst +++ b/docs/community/recommended.rst @@ -34,7 +34,7 @@ but do not belong in Requests proper. This library is actively maintained by members of the Requests core team, and reflects the functionality most requested by users within the community. -.. _Requests-Toolbelt: http://toolbelt.readthedocs.io/en/latest/index.html +.. _Requests-Toolbelt: https://toolbelt.readthedocs.io/en/latest/index.html Requests-Threads diff --git a/docs/conf.py b/docs/conf.py index 4bda98b0..503448d3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -376,4 +376,4 @@ epub_exclude_files = ['search.html'] # If false, no index is generated. #epub_use_index = True -intersphinx_mapping = {'urllib3': ('http://urllib3.readthedocs.io/en/latest', None)} +intersphinx_mapping = {'urllib3': ('https://urllib3.readthedocs.io/en/latest', None)} diff --git a/docs/dev/todo.rst b/docs/dev/todo.rst index 50b18155..707dea31 100644 --- a/docs/dev/todo.rst +++ b/docs/dev/todo.rst @@ -61,5 +61,5 @@ Requests currently supports the following versions of Python: Google AppEngine is not officially supported although support is available with the `Requests-Toolbelt`_. -.. _Requests-Toolbelt: http://toolbelt.readthedocs.io/ +.. _Requests-Toolbelt: https://toolbelt.readthedocs.io/ diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 587b3fdc..1bad6435 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -287,7 +287,7 @@ system. For the sake of security we recommend upgrading certifi frequently! .. _HTTP persistent connection: https://en.wikipedia.org/wiki/HTTP_persistent_connection -.. _connection pooling: http://urllib3.readthedocs.io/en/latest/reference/index.html#module-urllib3.connectionpool +.. _connection pooling: https://urllib3.readthedocs.io/en/latest/reference/index.html#module-urllib3.connectionpool .. _certifi: http://certifi.io/ .. _Mozilla trust store: https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt @@ -657,7 +657,7 @@ When you receive a response, Requests makes a guess at the encoding to use for decoding the response when you access the :attr:`Response.text ` attribute. Requests will first check for an encoding in the HTTP header, and if none is present, will use `chardet -`_ to attempt to guess the encoding. +`_ to attempt to guess the encoding. The only time Requests will not do this is if no explicit charset is present in the HTTP headers **and** the ``Content-Type`` @@ -884,7 +884,7 @@ Link Headers Many HTTP APIs feature Link headers. They make APIs more self describing and discoverable. -GitHub uses these for `pagination `_ +GitHub uses these for `pagination `_ in their API, for example:: >>> url = 'https://api.github.com/users/kennethreitz/repos?page=1&per_page=10' diff --git a/docs/user/authentication.rst b/docs/user/authentication.rst index 8ffab504..411f79fd 100644 --- a/docs/user/authentication.rst +++ b/docs/user/authentication.rst @@ -136,11 +136,11 @@ Further examples can be found under the `Requests organization`_ and in the .. _OAuth: http://oauth.net/ .. _requests_oauthlib: https://github.com/requests/requests-oauthlib -.. _requests-oauthlib OAuth2 documentation: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html -.. _Web Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#web-application-flow -.. _Mobile Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#mobile-application-flow -.. _Legacy Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#legacy-application-flow -.. _Backend Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#backend-application-flow +.. _requests-oauthlib OAuth2 documentation: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html +.. _Web Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#web-application-flow +.. _Mobile Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#mobile-application-flow +.. _Legacy Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#legacy-application-flow +.. _Backend Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#backend-application-flow .. _Kerberos: https://github.com/requests/requests-kerberos .. _NTLM: https://github.com/requests/requests-ntlm .. _Requests organization: https://github.com/requests From 620a5391c38a2eef86ba69748833aa3c00aaef62 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Wed, 10 Jan 2018 18:37:22 -0800 Subject: [PATCH 36/38] Remove unsupported Python 3.3 from tox.ini Python 3.3 is not a supported version so don't test it. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 2a961c82..38bf3ac4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] -envlist = py26,py27,py33,py34,py35,py36 +envlist = py26,py27,py34,py35,py36 [testenv] commands = pip install -e .[socks] - python setup.py test \ No newline at end of file + python setup.py test From 7cefa939f5a0b25d8792a949b7ae708412d62681 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Wed, 10 Jan 2018 18:33:06 -0800 Subject: [PATCH 37/38] Pass python_requires argument to setuptools Helps pip decide what version of the library to install. https://packaging.python.org/tutorials/distributing-packages/#python-requires > If your project only runs on certain Python versions, setting the > python_requires argument to the appropriate PEP 440 version specifier > string will prevent pip from installing the project on other Python > versions. https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords > python_requires > > A string corresponding to a version specifier (as defined in PEP 440) > for the Python version, used to specify the Requires-Python defined in > PEP 345. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e3bc8761..f32cca75 100755 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ setup( package_data={'': ['LICENSE', 'NOTICE'], 'requests': ['*.pem']}, package_dir={'requests': 'requests'}, include_package_data=True, + python_requires=">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", install_requires=requires, license=about['__license__'], zip_safe=False, From 2255c34a65b5b1353004dc8d49cc397cd794ec15 Mon Sep 17 00:00:00 2001 From: Darren Dormer Date: Tue, 12 Dec 2017 15:53:09 +0100 Subject: [PATCH 38/38] Fix DNS resolution by using hostname instead of netloc and strip username and password when comparing against proxy bypass items. --- AUTHORS.rst | 1 + HISTORY.rst | 3 ++- requests/utils.py | 19 +++++++++++-------- tests/test_utils.py | 25 +++++++++++++++++++++++-- 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 481ac6c7..2b4494ba 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -182,3 +182,4 @@ Patches and Suggestions - Arthur Vigil (`@ahvigil `_) - Nehal J Wani (`@nehaljwani `_) - Demetrios Bairaktaris (`@DemetriosBairaktaris `_) +- Darren Dormer (`@ddormer `_) diff --git a/HISTORY.rst b/HISTORY.rst index b099ecdb..db1d1f70 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,7 +17,8 @@ dev - Fixed issue where loading the default certificate bundle from a zip archive would raise an ``IOError`` - Fixed issue with unexpected ``ImportError`` on windows system which do not support ``winreg`` module - +- DNS resolution in proxy bypass no longer includes the username and password in + the request. This also fixes the issue of DNS queries failing on macOS. 2.18.4 (2017-08-15) +++++++++++++++++++ diff --git a/requests/utils.py b/requests/utils.py index 8c1b9bec..df18a0bf 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -703,28 +703,31 @@ def should_bypass_proxies(url, no_proxy): no_proxy_arg = no_proxy if no_proxy is None: no_proxy = get_proxy('no_proxy') - netloc = urlparse(url).netloc + parsed = urlparse(url) if no_proxy: # We need to check whether we match here. We need to see if we match - # the end of the netloc, both with and without the port. + # the end of the hostname, both with and without the port. no_proxy = ( host for host in no_proxy.replace(' ', '').split(',') if host ) - ip = netloc.split(':')[0] - if is_ipv4_address(ip): + if is_ipv4_address(parsed.hostname): for proxy_ip in no_proxy: if is_valid_cidr(proxy_ip): - if address_in_network(ip, proxy_ip): + if address_in_network(parsed.hostname, proxy_ip): return True - elif ip == proxy_ip: + elif parsed.hostname == proxy_ip: # If no_proxy ip was defined in plain IP notation instead of cidr notation & # matches the IP of the index return True else: + host_with_port = parsed.hostname + if parsed.port: + host_with_port += ':{0}'.format(parsed.port) + for host in no_proxy: - if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + if parsed.hostname.endswith(host) or host_with_port.endswith(host): # The URL does match something in no_proxy, so we don't want # to apply the proxies on this URL. return True @@ -737,7 +740,7 @@ def should_bypass_proxies(url, no_proxy): # legitimate problems. with set_environ('no_proxy', no_proxy_arg): try: - bypass = proxy_bypass(netloc) + bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): bypass = False diff --git a/tests/test_utils.py b/tests/test_utils.py index 01cabe23..f39cd67b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -614,6 +614,7 @@ def test_urldefragauth(url, expected): ('http://172.16.1.1/', True), ('http://172.16.1.1:5000/', True), ('http://localhost.localdomain:5000/v1.0/', True), + ('http://google.com:6000/', True), ('http://172.16.1.12/', False), ('http://172.16.1.12:5000/', False), ('http://google.com:5000/v1.0/', False), @@ -622,11 +623,31 @@ def test_should_bypass_proxies(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not """ - monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') - monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') + monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') + monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') assert should_bypass_proxies(url, no_proxy=None) == expected +@pytest.mark.parametrize( + 'url, expected', ( + ('http://172.16.1.1/', '172.16.1.1'), + ('http://172.16.1.1:5000/', '172.16.1.1'), + ('http://user:pass@172.16.1.1', '172.16.1.1'), + ('http://user:pass@172.16.1.1:5000', '172.16.1.1'), + ('http://hostname/', 'hostname'), + ('http://hostname:5000/', 'hostname'), + ('http://user:pass@hostname', 'hostname'), + ('http://user:pass@hostname:5000', 'hostname'), + )) +def test_should_bypass_proxies_pass_only_hostname(url, expected, mocker): + """The proxy_bypass function should be called with a hostname or IP without + a port number or auth credentials. + """ + proxy_bypass = mocker.patch('requests.utils.proxy_bypass') + should_bypass_proxies(url, no_proxy=None) + proxy_bypass.assert_called_once_with(expected) + + @pytest.mark.parametrize( 'cookiejar', ( compat.cookielib.CookieJar(),