From 48fe34447de02bfe9827321fafe3277741d77c38 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Thu, 20 Aug 2015 21:59:57 +0000 Subject: [PATCH 01/12] Implement per-host proxies This change allows the proxy dict to be have entries of the form {'://': ''}. Host-specific proxies will be used in preference to the scheme-specific proxies (i.e., proxy dict entries with keys like 'http' or 'https'). Fixes #2722 --- requests/adapters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/adapters.py b/requests/adapters.py index 60afb583..0f3dc4da 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -239,7 +239,8 @@ class HTTPAdapter(BaseAdapter): :param proxies: (optional) A Requests-style dictionary of proxies used on this request. """ proxies = proxies or {} - proxy = proxies.get(urlparse(url.lower()).scheme) + u = urlparse(url.lower()) + proxy = proxies.get(u.scheme+'://'+u.hostname, proxies.get(u.scheme)) if proxy: proxy = prepend_scheme_if_needed(proxy, 'http') From 56844d128754b923211a981618903c34fd36ed7b Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Tue, 25 Aug 2015 14:53:52 +0000 Subject: [PATCH 02/12] Add documentation for host-specific proxies --- docs/user/advanced.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 8466a4af..78fc6054 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -459,12 +459,18 @@ If you need to use a proxy, you can configure individual requests with the import requests proxies = { + "http://10.20.1.128": "http://10.10.1.10:5323", + "https://10.20.1.140": "http://10.10.1.10:6431" "http": "http://10.10.1.10:3128", "https": "http://10.10.1.10:1080", } requests.get("http://example.org", proxies=proxies) +A request's ``://`` is looked up in the proxies +dictionary first, to provide host-specific proxies. If no +host-specific proxy is found, the request's scheme is looked up. + You can also configure proxies by setting the environment variables ``HTTP_PROXY`` and ``HTTPS_PROXY``. From 618514231bd24d7eec7b8d486f4a55cb7ef8d522 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Tue, 25 Aug 2015 15:11:27 +0000 Subject: [PATCH 03/12] Reword documentation for per-host proxies --- docs/user/advanced.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 78fc6054..c0e6aabc 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -459,18 +459,12 @@ If you need to use a proxy, you can configure individual requests with the import requests proxies = { - "http://10.20.1.128": "http://10.10.1.10:5323", - "https://10.20.1.140": "http://10.10.1.10:6431" "http": "http://10.10.1.10:3128", "https": "http://10.10.1.10:1080", } requests.get("http://example.org", proxies=proxies) -A request's ``://`` is looked up in the proxies -dictionary first, to provide host-specific proxies. If no -host-specific proxy is found, the request's scheme is looked up. - You can also configure proxies by setting the environment variables ``HTTP_PROXY`` and ``HTTPS_PROXY``. @@ -488,8 +482,15 @@ To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` syn "http": "http://user:pass@10.10.1.10:3128/", } -Note that proxy URLs must include the scheme. +To give a proxy for a specific scheme and host, use the +`scheme://host` form for the key:: + proxies = { + "http://10.20.1.128": "http://10.10.1.10:5323", + } + +Note that proxy URLs must include the scheme. + .. _compliance: Compliance From 22075f02d0f41ee8d6e7328361b59c3b2d54bdaf Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Tue, 25 Aug 2015 18:21:20 +0000 Subject: [PATCH 04/12] Clarify the per-host proxy code --- requests/adapters.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 0f3dc4da..cc6c8971 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -239,8 +239,10 @@ class HTTPAdapter(BaseAdapter): :param proxies: (optional) A Requests-style dictionary of proxies used on this request. """ proxies = proxies or {} - u = urlparse(url.lower()) - proxy = proxies.get(u.scheme+'://'+u.hostname, proxies.get(u.scheme)) + urlparts = urlparse(url.lower()) + proxy = proxies.get(urlparts.scheme+'://'+urlparts.hostname) + if proxy is None: + proxy = proxies.get(urlparts.scheme) if proxy: proxy = prepend_scheme_if_needed(proxy, 'http') From d7a3aa618839e304c324e6092953131a1f43c647 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Thu, 27 Aug 2015 17:26:14 +0000 Subject: [PATCH 05/12] Fix another place the proxy-selecting logic is used. --- requests/adapters.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index cc6c8971..b9b3326e 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -278,10 +278,12 @@ class HTTPAdapter(BaseAdapter): :param proxies: A dictionary of schemes to proxy URLs. """ proxies = proxies or {} - scheme = urlparse(request.url).scheme - proxy = proxies.get(scheme) + urlparts = urlparse(request.url.lower()) + proxy = proxies.get(urlparts.scheme+'://'+urlparts.hostname) + if proxy is None: + proxy = proxies.get(urlparts.scheme) - if proxy and scheme != 'https': + if proxy and urlparts.scheme != 'https': url = urldefragauth(request.url) else: url = request.path_url From 2c8721540ad70cadb3c71912b1ab8577f2f8fd53 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Thu, 27 Aug 2015 17:26:45 +0000 Subject: [PATCH 06/12] Fix documentation for proxies in the Session class --- requests/sessions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index c3ef363c..a12f297b 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -299,9 +299,9 @@ class Session(SessionRedirectMixin): #: :class:`Request `. self.auth = None - #: Dictionary mapping protocol to the URL of the proxy (e.g. - #: {'http': 'foo.bar:3128'}) to be used on each - #: :class:`Request `. + #: Dictionary mapping protocol or protocol and host to the URL of the proxy + #: (e.g. {'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}) to + #: be used on each :class:`Request `. self.proxies = {} #: Event-handling hooks. @@ -428,8 +428,8 @@ class Session(SessionRedirectMixin): :type timeout: float or tuple :param allow_redirects: (optional) Set to True by default. :type allow_redirects: bool - :param proxies: (optional) Dictionary mapping protocol to the URL of - the proxy. + :param proxies: (optional) Dictionary mapping protocol or protocol and + hostname to the URL of the proxy. :param stream: (optional) whether to immediately download the response content. Defaults to ``False``. :param verify: (optional) if ``True``, the SSL cert will be verified. From 500f15da1fe54e3773be42f7770680d1f86b699f Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Thu, 27 Aug 2015 17:38:18 +0000 Subject: [PATCH 07/12] Factor out the proxy selection code --- requests/adapters.py | 18 +++++------------- requests/utils.py | 8 ++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index b9b3326e..f61ea824 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -17,7 +17,8 @@ from .packages.urllib3.util import Timeout as TimeoutSauce from .packages.urllib3.util.retry import Retry from .compat import urlparse, basestring from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, - prepend_scheme_if_needed, get_auth_from_url, urldefragauth) + prepend_scheme_if_needed, get_auth_from_url, urldefragauth, + select_proxy) from .structures import CaseInsensitiveDict from .packages.urllib3.exceptions import ConnectTimeoutError from .packages.urllib3.exceptions import HTTPError as _HTTPError @@ -238,11 +239,7 @@ class HTTPAdapter(BaseAdapter): :param url: The URL to connect to. :param proxies: (optional) A Requests-style dictionary of proxies used on this request. """ - proxies = proxies or {} - urlparts = urlparse(url.lower()) - proxy = proxies.get(urlparts.scheme+'://'+urlparts.hostname) - if proxy is None: - proxy = proxies.get(urlparts.scheme) + proxy = select_proxy(url, proxies) if proxy: proxy = prepend_scheme_if_needed(proxy, 'http') @@ -277,13 +274,8 @@ class HTTPAdapter(BaseAdapter): :param request: The :class:`PreparedRequest ` being sent. :param proxies: A dictionary of schemes to proxy URLs. """ - proxies = proxies or {} - urlparts = urlparse(request.url.lower()) - proxy = proxies.get(urlparts.scheme+'://'+urlparts.hostname) - if proxy is None: - proxy = proxies.get(urlparts.scheme) - - if proxy and urlparts.scheme != 'https': + proxy = select_proxy(request.url, proxies) + if proxy and not request.url.lower().startswith('https'): url = urldefragauth(request.url) else: url = request.path_url diff --git a/requests/utils.py b/requests/utils.py index 3fd0e41f..ddc2c00a 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -537,6 +537,14 @@ def get_environ_proxies(url): else: return getproxies() +def select_proxy(url, proxies): + """Select a proxy, if applicable.""" + proxies = proxies or {} + urlparts = urlparse(url.lower()) + proxy = proxies.get(urlparts.scheme+'://'+urlparts.hostname) + if proxy is None: + proxy = proxies.get(urlparts.scheme) + return proxy def default_user_agent(name="python-requests"): """Return a string representing the default user agent.""" From c80121df91997535f4c27f9c44c7aca10ad39af6 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Thu, 27 Aug 2015 17:41:12 +0000 Subject: [PATCH 08/12] Fix documentation for the proxies dictionary --- requests/adapters.py | 2 +- requests/utils.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index f61ea824..206978b3 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -272,7 +272,7 @@ class HTTPAdapter(BaseAdapter): :class:`HTTPAdapter `. :param request: The :class:`PreparedRequest ` being sent. - :param proxies: A dictionary of schemes to proxy URLs. + :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs. """ proxy = select_proxy(request.url, proxies) if proxy and not request.url.lower().startswith('https'): diff --git a/requests/utils.py b/requests/utils.py index ddc2c00a..95bf6b5f 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -538,7 +538,11 @@ def get_environ_proxies(url): return getproxies() def select_proxy(url, proxies): - """Select a proxy, if applicable.""" + """Select a proxy for the url, if applicable. + + :param url: The url being for the request + :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs + """ proxies = proxies or {} urlparts = urlparse(url.lower()) proxy = proxies.get(urlparts.scheme+'://'+urlparts.hostname) From fb496e4e86cafd78fd510e6d1ecf539c0545bcae Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Thu, 27 Aug 2015 18:20:11 +0000 Subject: [PATCH 09/12] Add test for per-host proxy selection --- test_requests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test_requests.py b/test_requests.py index 7e5e4d8f..c67e569e 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1325,6 +1325,15 @@ class UtilsTestCase(unittest.TestCase): 'http://localhost.localdomain:5000/v1.0/') == {} assert get_environ_proxies('http://www.requests.com/') != {} + def test_select_proxies(self): + """Make sure we can select per-host proxies correctly.""" + from requests.utils import select_proxy + proxies = {'http': 'http://http.proxy', + 'http://some.host': 'http://some.host.proxy'} + assert select_proxy('hTTp://u:p@Some.Host/path', proxies) == 'http://some.host.proxy' + assert select_proxy('hTTp://u:p@Other.Host/path', proxies) == 'http://http.proxy' + assert select_proxy('hTTps://Other.Host', proxies) is None + def test_guess_filename_when_int(self): from requests.utils import guess_filename assert None is guess_filename(1) From f3d4480a33b367cf020b0ed447bca10797706094 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 28 Aug 2015 20:06:45 +0000 Subject: [PATCH 10/12] Use url parsing to check the scheme --- requests/adapters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/adapters.py b/requests/adapters.py index 206978b3..96d64327 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -275,7 +275,8 @@ class HTTPAdapter(BaseAdapter): :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs. """ proxy = select_proxy(request.url, proxies) - if proxy and not request.url.lower().startswith('https'): + scheme = urlparse(request.url.lower()).scheme + if proxy and scheme != 'https': url = urldefragauth(request.url) else: url = request.path_url From 4c6540898c101a606e1a50e539f01a82163bdae0 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 28 Aug 2015 20:20:07 +0000 Subject: [PATCH 11/12] don't lowercase a url before urlparsing it urlparse automatically lowercases the scheme and hostname --- requests/adapters.py | 2 +- requests/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 96d64327..f911fc57 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -275,7 +275,7 @@ class HTTPAdapter(BaseAdapter): :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs. """ proxy = select_proxy(request.url, proxies) - scheme = urlparse(request.url.lower()).scheme + scheme = urlparse(request.url).scheme if proxy and scheme != 'https': url = urldefragauth(request.url) else: diff --git a/requests/utils.py b/requests/utils.py index 95bf6b5f..3d4c7945 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -544,7 +544,7 @@ def select_proxy(url, proxies): :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs """ proxies = proxies or {} - urlparts = urlparse(url.lower()) + urlparts = urlparse(url) proxy = proxies.get(urlparts.scheme+'://'+urlparts.hostname) if proxy is None: proxy = proxies.get(urlparts.scheme) From 704a922fc2d77bbc1b24fe2a02ad6876bdc16d5c Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Fri, 4 Sep 2015 22:55:38 +0000 Subject: [PATCH 12/12] Clarify documentation for per-host proxies Make sure it's clear that a per-host proxy matches a scheme and exact hostname. --- docs/user/advanced.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index c0e6aabc..3b060a0e 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -483,7 +483,10 @@ To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` syn } To give a proxy for a specific scheme and host, use the -`scheme://host` form for the key:: +`scheme://hostname` form for the key. This will match for +any request to the given scheme and exact hostname. + +:: proxies = { "http://10.20.1.128": "http://10.10.1.10:5323",