diff --git a/AUTHORS.rst b/AUTHORS.rst
index 6c2e39c5..668d4580 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -180,3 +180,5 @@ Patches and Suggestions
- Andrii Soldatenko (`@a_soldatenko `_)
- Moinuddin Quadri (`@moin18 `_)
- Matt Kohl (`@mattkohl `_)
+- Jonathan Vanasco (`@jvanasco `_)
+
diff --git a/HISTORY.rst b/HISTORY.rst
index 5ec8bb21..c26036c1 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,6 +3,15 @@
Release History
---------------
+**Unreleased**
++++++++++++++++++++
+
+- The behavior of ``SessionRedirectMixin`` was slightly altered.
+ ``resolve_redirects`` will now detect a redirect by calling
+ ``get_redirect_target(response)`` instead of directly
+ querying ``Response.is_redirect`` and ``Response.headers['location']``.
+ Advanced users will be able to process malformed redirects more easily.
+
2.13.0 (2017-01-24)
+++++++++++++++++++
diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html
index 457688d9..fe113734 100644
--- a/docs/_templates/sidebarintro.html
+++ b/docs/_templates/sidebarintro.html
@@ -14,6 +14,17 @@
human beings.
+
+The Hitchhiker's Guide to Python
+
+This guide is now available in tangible book form!
+
+
+
+All proceeds are being directly donated to the DjangoGirls organization.
+
+
+
Stay Informed
Receive updates on new releases and upcoming projects.
@@ -24,10 +35,12 @@
Say Thanks!
Join Mailing List.
+
Other Projects
More Kenneth Reitz projects:
+ - edmsynths.com
- pipenv
- pep8.org
- httpbin.org
diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html
index 798f8091..2fb8062b 100644
--- a/docs/_templates/sidebarlogo.html
+++ b/docs/_templates/sidebarlogo.html
@@ -21,6 +21,15 @@
+The Hitchhiker's Guide to Python
+
+This guide is now available in tangible book form!
+
+
+
+All proceeds are being directly donated to the DjangoGirls organization.
+
+
If you enjoy using this project, Say Thanks!
+
Other Projects
More Kenneth Reitz projects:
+ - edmsynths.com
- pipenv
- pep8.org
- httpbin.org
diff --git a/requests/sessions.py b/requests/sessions.py
index 74080b28..e74fd32c 100644
--- a/requests/sessions.py
+++ b/requests/sessions.py
@@ -87,6 +87,15 @@ def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict):
class SessionRedirectMixin(object):
+ def get_redirect_target(self, resp):
+ """Receives a Response. Returns a redirect URI or ``None``"""
+ if resp.is_redirect:
+ if not is_valid_location(response):
+ raise InvalidHeader('Response contains multiple Location headers. '
+ 'Unable to perform redirect.')
+ return resp.headers['location']
+ return None
+
def resolve_redirects(self, response, request, stream=False, timeout=None,
verify=True, cert=None, proxies=None, **adapter_kwargs):
"""Given a Response, yields Responses until 'Location' header-based
@@ -97,61 +106,52 @@ class SessionRedirectMixin(object):
redirect_count = 0
history = [] # keep track of history
- while response.is_redirect:
- if not is_valid_location(response):
- raise InvalidHeader('Response contains multiple Location headers. '
- 'Unable to perform redirect.')
+ url = self.get_redirect_target(response)
+ while url:
prepared_request = request.copy()
- if redirect_count > 0:
-
- # Store this Response in local history.
- history.append(response)
-
- # Copy local history to Response.history.
- response.history = list(history)
+ # Update history and keep track of redirects.
+ # response.history must ignore the original request in this loop
+ hist.append(response)
+ response.history = hist[1:]
try:
response.content # Consume socket so it can be released
except (ChunkedEncodingError, ConnectionError, ContentDecodingError, RuntimeError):
response.raw.read(decode_content=False)
- # Don't exceed configured Session.max_redirects.
- if redirect_count >= self.max_redirects:
+ if len(response.history) >= self.max_redirects:
raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=response)
# Release the connection back into the pool.
response.close()
- location_url = response.headers['location']
- method = request.method
-
# Handle redirection without scheme (see: RFC 1808 Section 4)
- if location_url.startswith('//'):
+ if url.startswith('//'):
parsed_rurl = urlparse(response.url)
- location_url = '%s:%s' % (parsed_rurl.scheme, location_url)
+ location_url = '%s:%s' % (parsed_rurl.scheme, url)
# The scheme should be lower case...
- parsed = urlparse(location_url)
+ parsed = urlparse(url)
location_url = parsed.geturl()
# On Python 3, the location header was decoded using Latin 1, but
# urlparse in requote_uri will encode it with UTF-8 before quoting.
# Because of this insanity, we need to fix it up ourselves by
# sending the URL back to bytes ourselves.
- if is_py3 and isinstance(location_url, str):
- location_url = location_url.encode('latin1')
+ if is_py3 and isinstance(url, str):
+ url = url.encode('latin1')
# Facilitate relative 'location' headers, as allowed by RFC 7231.
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
# Compliant with RFC3986, we percent encode the url.
if not parsed.netloc:
- location_url = urljoin(response.url, requote_uri(location_url))
+ url = urljoin(response.url, requote_uri(url))
else:
- location_url = requote_uri(location_url)
+ url = requote_uri(url)
- prepared_request.url = to_native_string(location_url)
+ prepared_request.url = to_native_string(url)
# Cache the url, unless it redirects to itself.
if response.is_permanent_redirect and request.url != prepared_request.url:
self.redirect_cache[request.url] = prepared_request.url
@@ -212,7 +212,8 @@ class SessionRedirectMixin(object):
extract_cookies_to_jar(self.cookies, prepared_request, response.raw)
- redirect_count += 1
+ # extract redirect url, if any, for the next loop
+ url = self.get_redirect_target(response)
yield response
def rebuild_auth(self, prepared_request, response):
@@ -252,13 +253,16 @@ class SessionRedirectMixin(object):
:rtype: dict
"""
+ proxies = proxies if proxies is not None else {}
headers = prepared_request.headers
url = prepared_request.url
scheme = urlparse(url).scheme
- new_proxies = proxies.copy() if proxies is not None else {}
+ new_proxies = proxies.copy()
+ no_proxy = proxies.get('no_proxy')
- if self.trust_env and not should_bypass_proxies(url):
- environ_proxies = get_environ_proxies(url)
+ bypass_proxy = should_bypass_proxies(url, no_proxy=no_proxy)
+ if self.trust_env and not bypass_proxy:
+ environ_proxies = get_environ_proxies(url, no_proxy=no_proxy)
proxy = environ_proxies.get(scheme, environ_proxies.get('all'))
@@ -696,10 +700,14 @@ class Session(SessionRedirectMixin):
# can delete proxy information, which can then be re-added by a more
# specific layer. So we begin by getting the environment's proxies,
# then add the Session, then add the request.
+ no_proxy = proxies.get('no_proxy') if proxies is not None else None
+ if no_proxy is None:
+ no_proxy = self.proxies.get('no_proxy')
+
env_proxies = {}
if self.trust_env:
- env_proxies = get_environ_proxies(url) or {}
+ env_proxies = get_environ_proxies(url, no_proxy=no_proxy) or {}
new_proxies = merge_setting(self.proxies, env_proxies)
proxies = merge_setting(proxies, new_proxies)
diff --git a/requests/utils.py b/requests/utils.py
index 02fb22bd..e9460be4 100644
--- a/requests/utils.py
+++ b/requests/utils.py
@@ -11,6 +11,7 @@ that are also useful for external consumption.
import cgi
import codecs
import collections
+import contextlib
import io
import os
import re
@@ -91,14 +92,16 @@ def super_len(o):
else:
if hasattr(o, 'seek') and total_length is None:
# StringIO and BytesIO have seek but no useable fileno
+ try:
+ # seek to end of file
+ o.seek(0, 2)
+ total_length = o.tell()
- # seek to end of file
- o.seek(0, 2)
- total_length = o.tell()
-
- # seek back to current position to support
- # partially read file-like objects
- o.seek(current_position or 0)
+ # seek back to current position to support
+ # partially read file-like objects
+ o.seek(current_position or 0)
+ except (OSError, IOError):
+ total_length = 0
if total_length is None:
total_length = 0
@@ -568,7 +571,29 @@ def is_valid_cidr(string_network):
return True
-def should_bypass_proxies(url):
+@contextlib.contextmanager
+def set_environ(env_name, value):
+ """Set the environment variable 'env_name' to 'value'
+
+ Save previous value, yield, and then restore the previous value stored in
+ the environment variable 'env_name'.
+
+ If 'value' is None, do nothing"""
+ if value is not None:
+ old_value = os.environ.get(env_name)
+ os.environ[env_name] = value
+ try:
+ yield
+ finally:
+ if value is None:
+ return
+ if old_value is None:
+ del os.environ[env_name]
+ else:
+ os.environ[env_name] = old_value
+
+
+def should_bypass_proxies(url, no_proxy):
"""
Returns whether we should bypass proxies or not.
@@ -578,7 +603,9 @@ def should_bypass_proxies(url):
# First check whether no_proxy is defined. If it is, check that the URL
# we're getting isn't in the no_proxy list.
- no_proxy = get_proxy('no_proxy')
+ no_proxy_arg = no_proxy
+ if no_proxy is None:
+ no_proxy = get_proxy('no_proxy')
netloc = urlparse(url).netloc
if no_proxy:
@@ -611,10 +638,11 @@ def should_bypass_proxies(url):
# of Python 2.6, so allow this call to fail. Only catch the specific
# exceptions we've seen, though: this call failing in other ways can reveal
# legitimate problems.
- try:
- bypass = proxy_bypass(netloc)
- except (TypeError, socket.gaierror):
- bypass = False
+ with set_environ('no_proxy', no_proxy_arg):
+ try:
+ bypass = proxy_bypass(netloc)
+ except (TypeError, socket.gaierror):
+ bypass = False
if bypass:
return True
@@ -622,13 +650,13 @@ def should_bypass_proxies(url):
return False
-def get_environ_proxies(url):
+def get_environ_proxies(url, no_proxy):
"""
Return a dict of environment proxies.
:rtype: dict
"""
- if should_bypass_proxies(url):
+ if should_bypass_proxies(url, no_proxy=no_proxy):
return {}
else:
return getproxies()
diff --git a/tests/test_requests.py b/tests/test_requests.py
index b6cda0a0..696cb2bd 100755
--- a/tests/test_requests.py
+++ b/tests/test_requests.py
@@ -1650,7 +1650,7 @@ class TestRequests:
def tell(self):
return 0
- def seek(self, pos):
+ def seek(self, pos, whence=0):
raise OSError()
def __iter__(self):
@@ -1924,6 +1924,49 @@ class TestRequests:
assert 'Transfer-Encoding' in prepared_request.headers
assert 'Content-Length' not in prepared_request.headers
+ def test_custom_redirect_mixin(self, httpbin):
+ """Tests a custom mixin to overwrite ``get_redirect_target``.
+
+ Ensures a subclassed ``requests.Session`` can handle a certain type of
+ malformed redirect responses.
+
+ 1. original request receives a proper response: 302 redirect
+ 2. following the redirect, a malformed response is given:
+ status code = HTTP 200
+ location = alternate url
+ 3. the custom session catches the edge case and follows the redirect
+ """
+ url_final = httpbin('html')
+ querystring_malformed = urlencode({'location': url_final})
+ url_redirect_malformed = httpbin('response-headers?%s' % querystring_malformed)
+ querystring_redirect = urlencode({'url': url_redirect_malformed})
+ url_redirect = httpbin('redirect-to?%s' % querystring_redirect)
+ urls_test = [url_redirect,
+ url_redirect_malformed,
+ url_final,
+ ]
+
+ class CustomRedirectSession(requests.Session):
+ def get_redirect_target(self, resp):
+ # default behavior
+ if resp.is_redirect:
+ return resp.headers['location']
+ # edge case - check to see if 'location' is in headers anyways
+ location = resp.headers.get('location')
+ if location and (location != resp.url):
+ return location
+ return None
+
+ session = CustomRedirectSession()
+ r = session.get(urls_test[0])
+ assert len(r.history) == 2
+ assert r.status_code == 200
+ assert r.history[0].status_code == 302
+ assert r.history[0].is_redirect
+ assert r.history[1].status_code == 200
+ assert not r.history[1].is_redirect
+ assert r.url == urls_test[2]
+
class TestCaseInsensitiveDict:
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 1edf6218..11ebf617 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -161,7 +161,7 @@ class TestGetEnvironProxies:
'http://localhost.localdomain:5000/v1.0/',
))
def test_bypass(self, url):
- assert get_environ_proxies(url) == {}
+ assert get_environ_proxies(url, no_proxy=None) == {}
@pytest.mark.parametrize(
'url', (
@@ -170,7 +170,32 @@ class TestGetEnvironProxies:
'http://www.requests.com/',
))
def test_not_bypass(self, url):
- assert get_environ_proxies(url) != {}
+ assert get_environ_proxies(url, no_proxy=None) != {}
+
+ @pytest.mark.parametrize(
+ 'url', (
+ 'http://192.168.1.1:5000/',
+ 'http://192.168.1.1/',
+ 'http://www.requests.com/',
+ ))
+ def test_bypass_no_proxy_keyword(self, url):
+ no_proxy = '192.168.1.1,requests.com'
+ assert get_environ_proxies(url, no_proxy=no_proxy) == {}
+
+ @pytest.mark.parametrize(
+ 'url', (
+ 'http://192.168.0.1:5000/',
+ 'http://192.168.0.1/',
+ 'http://172.16.1.1/',
+ 'http://172.16.1.1:5000/',
+ 'http://localhost.localdomain:5000/v1.0/',
+ ))
+ def test_not_bypass_no_proxy_keyword(self, url, monkeypatch):
+ # This is testing that the 'no_proxy' argument overrides the
+ # environment variable 'no_proxy'
+ monkeypatch.setenv('http_proxy', 'http://proxy.example.com:3128/')
+ no_proxy = '192.168.1.1,requests.com'
+ assert get_environ_proxies(url, no_proxy=no_proxy) != {}
class TestIsIPv4Address:
@@ -525,7 +550,7 @@ def test_should_bypass_proxies(url, expected, monkeypatch):
"""
monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
- assert should_bypass_proxies(url) == expected
+ assert should_bypass_proxies(url, no_proxy=None) == expected
@pytest.mark.parametrize(
@@ -553,3 +578,24 @@ def test_add_dict_to_cookiejar(cookiejar):
)
def test_unicode_is_ascii(value, expected):
assert unicode_is_ascii(value) is expected
+
+
+@pytest.mark.parametrize(
+ 'url, expected', (
+ ('http://192.168.0.1:5000/', True),
+ ('http://192.168.0.1/', True),
+ ('http://172.16.1.1/', True),
+ ('http://172.16.1.1:5000/', True),
+ ('http://localhost.localdomain:5000/v1.0/', True),
+ ('http://172.16.1.12/', False),
+ ('http://172.16.1.12:5000/', False),
+ ('http://google.com:5000/v1.0/', False),
+ ))
+def test_should_bypass_proxies_no_proxy(
+ url, expected, monkeypatch):
+ """Tests for function should_bypass_proxies to check if proxy
+ can be bypassed or not using the 'no_proxy' argument
+ """
+ no_proxy = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1'
+ # Test 'no_proxy' argument
+ assert should_bypass_proxies(url, no_proxy=no_proxy) == expected