From 85400d8d6751071ef78f042d1efa72bdcf76cc0e Mon Sep 17 00:00:00 2001 From: "John L. Villalovos" Date: Tue, 6 Sep 2016 16:24:35 -0700 Subject: [PATCH 1/7] Allow use of 'no_proxy' in the proxies argument Add the ability to add 'no_proxy' and a value to the 'proxies' dictionary argument. https://github.com/kennethreitz/requests/issues/2817 Closes gh-2817 --- requests/sessions.py | 12 ++++++---- requests/utils.py | 42 ++++++++++++++++++++++++++++------- tests/test_utils.py | 52 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 91 insertions(+), 15 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 7983282a..51bad2ca 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -231,13 +231,16 @@ class SessionRedirectMixin(object): :rtype: dict """ + proxies = proxies if proxies is not None else {} headers = prepared_request.headers url = prepared_request.url scheme = urlparse(url).scheme - new_proxies = proxies.copy() if proxies is not None else {} + new_proxies = proxies.copy() + no_proxy = proxies.get('no_proxy') - if self.trust_env and not should_bypass_proxies(url): - environ_proxies = get_environ_proxies(url) + bypass_proxy = should_bypass_proxies(url, no_proxy=no_proxy) + if self.trust_env and not bypass_proxy: + environ_proxies = get_environ_proxies(url, no_proxy=no_proxy) proxy = environ_proxies.get(scheme, environ_proxies.get('all')) @@ -651,7 +654,8 @@ class Session(SessionRedirectMixin): # Gather clues from the surrounding environment. if self.trust_env: # Set environment's proxies. - env_proxies = get_environ_proxies(url) or {} + no_proxy = proxies.get('no_proxy') if proxies is not None else None + env_proxies = get_environ_proxies(url, no_proxy=no_proxy) for (k, v) in env_proxies.items(): proxies.setdefault(k, v) diff --git a/requests/utils.py b/requests/utils.py index 47325090..e5ecd350 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -11,6 +11,7 @@ that are also useful for external consumption. import cgi import codecs import collections +import contextlib import io import os import re @@ -554,7 +555,29 @@ def is_valid_cidr(string_network): return True -def should_bypass_proxies(url): +@contextlib.contextmanager +def set_environ(env_name, value): + """Set the environment variable 'env_name' to 'value' + + Save previous value, yield, and then restore the previous value stored in + the environment variable 'env_name'. + + If 'value' is None, do nothing""" + if value is not None: + old_value = os.environ.get(env_name) + os.environ[env_name] = value + try: + yield + finally: + if value is None: + return + if old_value is None: + del os.environ[env_name] + else: + os.environ[env_name] = old_value + + +def should_bypass_proxies(url, no_proxy): """ Returns whether we should bypass proxies or not. @@ -564,7 +587,9 @@ def should_bypass_proxies(url): # First check whether no_proxy is defined. If it is, check that the URL # we're getting isn't in the no_proxy list. - no_proxy = get_proxy('no_proxy') + no_proxy_arg = no_proxy + if no_proxy is None: + no_proxy = get_proxy('no_proxy') netloc = urlparse(url).netloc if no_proxy: @@ -597,10 +622,11 @@ def should_bypass_proxies(url): # of Python 2.6, so allow this call to fail. Only catch the specific # exceptions we've seen, though: this call failing in other ways can reveal # legitimate problems. - try: - bypass = proxy_bypass(netloc) - except (TypeError, socket.gaierror): - bypass = False + with set_environ('no_proxy', no_proxy_arg): + try: + bypass = proxy_bypass(netloc) + except (TypeError, socket.gaierror): + bypass = False if bypass: return True @@ -608,13 +634,13 @@ def should_bypass_proxies(url): return False -def get_environ_proxies(url): +def get_environ_proxies(url, no_proxy): """ Return a dict of environment proxies. :rtype: dict """ - if should_bypass_proxies(url): + if should_bypass_proxies(url, no_proxy=no_proxy): return {} else: return getproxies() diff --git a/tests/test_utils.py b/tests/test_utils.py index 1edf6218..11ebf617 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -161,7 +161,7 @@ class TestGetEnvironProxies: 'http://localhost.localdomain:5000/v1.0/', )) def test_bypass(self, url): - assert get_environ_proxies(url) == {} + assert get_environ_proxies(url, no_proxy=None) == {} @pytest.mark.parametrize( 'url', ( @@ -170,7 +170,32 @@ class TestGetEnvironProxies: 'http://www.requests.com/', )) def test_not_bypass(self, url): - assert get_environ_proxies(url) != {} + assert get_environ_proxies(url, no_proxy=None) != {} + + @pytest.mark.parametrize( + 'url', ( + 'http://192.168.1.1:5000/', + 'http://192.168.1.1/', + 'http://www.requests.com/', + )) + def test_bypass_no_proxy_keyword(self, url): + no_proxy = '192.168.1.1,requests.com' + assert get_environ_proxies(url, no_proxy=no_proxy) == {} + + @pytest.mark.parametrize( + 'url', ( + 'http://192.168.0.1:5000/', + 'http://192.168.0.1/', + 'http://172.16.1.1/', + 'http://172.16.1.1:5000/', + 'http://localhost.localdomain:5000/v1.0/', + )) + def test_not_bypass_no_proxy_keyword(self, url, monkeypatch): + # This is testing that the 'no_proxy' argument overrides the + # environment variable 'no_proxy' + monkeypatch.setenv('http_proxy', 'http://proxy.example.com:3128/') + no_proxy = '192.168.1.1,requests.com' + assert get_environ_proxies(url, no_proxy=no_proxy) != {} class TestIsIPv4Address: @@ -525,7 +550,7 @@ def test_should_bypass_proxies(url, expected, monkeypatch): """ monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') - assert should_bypass_proxies(url) == expected + assert should_bypass_proxies(url, no_proxy=None) == expected @pytest.mark.parametrize( @@ -553,3 +578,24 @@ def test_add_dict_to_cookiejar(cookiejar): ) def test_unicode_is_ascii(value, expected): assert unicode_is_ascii(value) is expected + + +@pytest.mark.parametrize( + 'url, expected', ( + ('http://192.168.0.1:5000/', True), + ('http://192.168.0.1/', True), + ('http://172.16.1.1/', True), + ('http://172.16.1.1:5000/', True), + ('http://localhost.localdomain:5000/v1.0/', True), + ('http://172.16.1.12/', False), + ('http://172.16.1.12:5000/', False), + ('http://google.com:5000/v1.0/', False), + )) +def test_should_bypass_proxies_no_proxy( + url, expected, monkeypatch): + """Tests for function should_bypass_proxies to check if proxy + can be bypassed or not using the 'no_proxy' argument + """ + no_proxy = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' + # Test 'no_proxy' argument + assert should_bypass_proxies(url, no_proxy=no_proxy) == expected From 70f31a3166c1f9470b5cfad888f828357c1daadd Mon Sep 17 00:00:00 2001 From: jonathan vanasco Date: Fri, 10 Feb 2017 13:53:23 -0500 Subject: [PATCH 2/7] * initial attempt at `get_redirect_target` * removing the `i` from the redirect detection while-loop --- AUTHORS.rst | 2 ++ HISTORY.rst | 9 +++++++++ requests/sessions.py | 27 +++++++++++++++----------- tests/test_requests.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 11 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index d29fa812..48cd155b 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -177,3 +177,5 @@ Patches and Suggestions - Andrii Soldatenko (`@a_soldatenko `_) - Moinuddin Quadri (`@moin18 `_) - Matt Kohl (`@mattkohl `_) +- Jonathan Vanasco (`@jvanasco `_) + diff --git a/HISTORY.rst b/HISTORY.rst index 5ec8bb21..c26036c1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,15 @@ Release History --------------- +**Unreleased** ++++++++++++++++++++ + +- The behavior of ``SessionRedirectMixin`` was slightly altered. + ``resolve_redirects`` will now detect a redirect by calling + ``get_redirect_target(response)`` instead of directly + querying ``Response.is_redirect`` and ``Response.headers['location']``. + Advanced users will be able to process malformed redirects more easily. + 2.13.0 (2017-01-24) +++++++++++++++++++ diff --git a/requests/sessions.py b/requests/sessions.py index 7983282a..72ef179f 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -86,35 +86,39 @@ def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict): class SessionRedirectMixin(object): + + def get_redirect_target(self, resp): + """Receives a Response. Returns a redirect URI or ``None``""" + if resp.is_redirect: + return resp.headers['location'] + return None + def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, **adapter_kwargs): """Receives a Response. Returns a generator of Responses.""" - i = 0 hist = [] # keep track of history - while resp.is_redirect: + url = self.get_redirect_target(resp) + while url: prepared_request = req.copy() - if i > 0: - # Update history and keep track of redirects. - hist.append(resp) - new_hist = list(hist) - resp.history = new_hist + # Update history and keep track of redirects. + # resp.history must ignore the original request in this loop + hist.append(resp) + resp.history = hist[1:] try: resp.content # Consume socket so it can be released except (ChunkedEncodingError, ContentDecodingError, RuntimeError): resp.raw.read(decode_content=False) - if i >= self.max_redirects: + if len(resp.history) >= self.max_redirects: raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=resp) # Release the connection back into the pool. resp.close() - url = resp.headers['location'] - # Handle redirection without scheme (see: RFC 1808 Section 4) if url.startswith('//'): parsed_rurl = urlparse(resp.url) @@ -192,7 +196,8 @@ class SessionRedirectMixin(object): extract_cookies_to_jar(self.cookies, prepared_request, resp.raw) - i += 1 + # extract redirect url, if any, for the next loop + url = self.get_redirect_target(resp) yield resp def rebuild_auth(self, prepared_request, response): diff --git a/tests/test_requests.py b/tests/test_requests.py index fd35103b..cd4c68db 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1740,6 +1740,49 @@ class TestRequests: assert 'Transfer-Encoding' in prepared_request.headers assert 'Content-Length' not in prepared_request.headers + def test_custom_redirect_mixin(self, httpbin): + """Tests a custom mixin to overwrite ``get_redirect_target``. + + Ensures a subclassed ``requests.Session`` can handle a certain type of + malformed redirect responses. + + 1. original request receives a proper response: 302 redirect + 2. following the redirect, a malformed response is given: + status code = HTTP 200 + location = alternate url + 3. the custom session catches the edge case and follows the redirect + """ + url_final = httpbin('html') + querystring_malformed = urlencode({'location': url_final}) + url_redirect_malformed = httpbin('response-headers?%s' % querystring_malformed) + querystring_redirect = urlencode({'url': url_redirect_malformed}) + url_redirect = httpbin('redirect-to?%s' % querystring_redirect) + urls_test = [url_redirect, + url_redirect_malformed, + url_final, + ] + + class CustomRedirectSession(requests.Session): + def get_redirect_target(self, resp): + # default behavior + if resp.is_redirect: + return resp.headers['location'] + # edge case - check to see if 'location' is in headers anyways + location = resp.headers.get('location') + if location and (location != resp.url): + return location + return None + + session = CustomRedirectSession() + r = session.get(urls_test[0]) + assert len(r.history) == 2 + assert r.status_code == 200 + assert r.history[0].status_code == 302 + assert r.history[0].is_redirect + assert r.history[1].status_code == 200 + assert not r.history[1].is_redirect + assert r.url == urls_test[2] + class TestCaseInsensitiveDict: From e7b7574cfe0df12c19193757c0493582818cf5fb Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 11 Feb 2017 15:12:04 -0500 Subject: [PATCH 3/7] hhg2p --- docs/_templates/sidebarintro.html | 8 ++++++++ docs/_templates/sidebarlogo.html | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index 457688d9..d325d845 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -24,6 +24,14 @@

Say Thanks!

Join Mailing List.

+

Hitchhiker's Guide to Python

+ +

This guide is now available in tangible book form!

+ + + +

All proceeds are being directly +

Other Projects

More Kenneth Reitz projects:

diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 798f8091..271af911 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -28,6 +28,15 @@

+

Hitchhiker's Guide to Python

+ +

This guide is now available in tangible book form!

+ + + +

All proceeds are being directly donated to the DjangoGirls organization.

+ +

Other Projects

More Kenneth Reitz projects:

From f2fe7358469cf844bce04bf04478b8f95fd634f9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 11 Feb 2017 15:16:09 -0500 Subject: [PATCH 4/7] fixes --- docs/_templates/sidebarintro.html | 5 +++-- docs/_templates/sidebarlogo.html | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index d325d845..e2aebc9f 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -24,13 +24,14 @@

Say Thanks!

Join Mailing List.

-

Hitchhiker's Guide to Python

+

The Hitchhiker's Guide to Python

This guide is now available in tangible book form!

-

All proceeds are being directly +

All proceeds are being directly donated to the DjangoGirls organization.

+

Other Projects

diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 271af911..0c8bccf8 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -28,7 +28,7 @@

-

Hitchhiker's Guide to Python

+

The Hitchhiker's Guide to Python

This guide is now available in tangible book form!

From 6cbebf782a4309010eef3bba088867a206f617b4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 11 Feb 2017 15:17:27 -0500 Subject: [PATCH 5/7] fixes --- docs/_templates/sidebarintro.html | 19 +++++++++++-------- docs/_templates/sidebarlogo.html | 15 ++++++++------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index e2aebc9f..fc9a659a 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -14,6 +14,17 @@ human beings.

+ +

The Hitchhiker's Guide to Python

+ +

This guide is now available in tangible book form!

+ + + +

All proceeds are being directly donated to the DjangoGirls organization.

+ + +

Stay Informed

Receive updates on new releases and upcoming projects.

@@ -24,14 +35,6 @@

Say Thanks!

Join Mailing List.

-

The Hitchhiker's Guide to Python

- -

This guide is now available in tangible book form!

- - - -

All proceeds are being directly donated to the DjangoGirls organization.

-

Other Projects

diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 0c8bccf8..4c05a538 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -21,13 +21,6 @@
-

If you enjoy using this project, Say Thanks!

- -

- -

-

The Hitchhiker's Guide to Python

This guide is now available in tangible book form!

@@ -37,6 +30,14 @@

All proceeds are being directly donated to the DjangoGirls organization.

+

If you enjoy using this project, Say Thanks!

+ +

+ +

+ +

Other Projects

More Kenneth Reitz projects:

From 90f3842ed60f91ff101189f792ce21f7c762b1a0 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 12 Feb 2017 02:18:50 -0500 Subject: [PATCH 6/7] edmsynths --- docs/_templates/sidebarintro.html | 1 + docs/_templates/sidebarlogo.html | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index fc9a659a..fe113734 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -40,6 +40,7 @@

More Kenneth Reitz projects:

    +
  • edmsynths.com
  • pipenv
  • pep8.org
  • httpbin.org
    • diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 4c05a538..2fb8062b 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -42,6 +42,7 @@

    More Kenneth Reitz projects:

      +
    • edmsynths.com
    • pipenv
    • pep8.org
    • httpbin.org
      • From f47aff68f1352c3a5030fe70bbdf7c00171e207b Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Tue, 14 Feb 2017 07:45:41 -0700 Subject: [PATCH 7/7] properly handled failed seek --- requests/utils.py | 16 +++++++++------- tests/test_requests.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index e5ecd350..6365034c 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -92,14 +92,16 @@ def super_len(o): else: if hasattr(o, 'seek') and total_length is None: # StringIO and BytesIO have seek but no useable fileno + try: + # seek to end of file + o.seek(0, 2) + total_length = o.tell() - # seek to end of file - o.seek(0, 2) - total_length = o.tell() - - # seek back to current position to support - # partially read file-like objects - o.seek(current_position or 0) + # seek back to current position to support + # partially read file-like objects + o.seek(current_position or 0) + except (OSError, IOError): + total_length = 0 if total_length is None: total_length = 0 diff --git a/tests/test_requests.py b/tests/test_requests.py index cd4c68db..26d4951e 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1546,7 +1546,7 @@ class TestRequests: def tell(self): return 0 - def seek(self, pos): + def seek(self, pos, whence=0): raise OSError() def __iter__(self):