kennethreitz/requests
1
0
Fork 0
mirror of https://github.com/kennethreitz/requests.git synced 2026-06-05 22:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into proposed/3.0.0

Browse Source
This commit is contained in:
Cory Benfield
2017-02-14 15:59:53 +00:00
parent 250b3e5c8c 2414fa82c8
commit 56dd77fc52
8 changed files with 208 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AUTHORS.rst
+2
View File
@@ -180,3 +180,5 @@ Patches and Suggestions
- Andrii Soldatenko (`@a_soldatenko <https://github.com/andriisoldatenko>`_)
- Moinuddin Quadri <moin18@gmail.com> (`@moin18 <https://github.com/moin18>`_)
- Matt Kohl (`@mattkohl <https://github.com/mattkohl>`_)
- Jonathan Vanasco (`@jvanasco <https://github.com/jvanasco>`_)
HISTORY.rst
+9
View File
@@ -3,6 +3,15 @@
Release History
---------------
**Unreleased**
+++++++++++++++++++
- The behavior of ``SessionRedirectMixin`` was slightly altered.
``resolve_redirects`` will now detect a redirect by calling
``get_redirect_target(response)`` instead of directly
querying ``Response.is_redirect`` and ``Response.headers['location']``.
Advanced users will be able to process malformed redirects more easily.
2.13.0 (2017-01-24)
+++++++++++++++++++
docs/_templates/sidebarintro.html
Vendored
+13
View File
@@ -14,6 +14,17 @@
human beings.
</p>
<h3>The Hitchhiker's Guide to Python</h3>
<p>This guide is now available in tangible book form!</p>
<a href="https://www.amazon.com/Hitchhikers-Guide-Python-Practices-Development/dp/1491933178/ref=as_li_ss_il?ie=UTF8&linkCode=li2&tag=bookforkind-20&linkId=804806ebdacaf3b56567347f3afbdbca" target="_blank"><img border="0" src="https://ws-na.amazon-adsystem.com/widgets/q?_encoding=UTF8&ASIN=1491933178&Format=_SL160_&ID=AsinImage&MarketPlace=US&ServiceVersion=20070822&WS=1&tag=bookforkind-20" ></a><img src="//ir-na.amazon-adsystem.com/e/ir?t=bookforkind-20&l=li2&o=1&a=1491933178" width="1" height="1" border="0" alt="" style="border:none !important; margin:0px !important;" />
<p>All proceeds are being directly donated to the <a href="https://djangogirls.org">DjangoGirls</a> organization.</p>
<h3>Stay Informed</h3>
<p>Receive updates on new releases and upcoming projects.</p>
@@ -24,10 +35,12 @@
<p><a href="https://saythanks.io/to/kennethreitz">Say Thanks!</a></p>
<p><a href="http://tinyletter.com/kennethreitz">Join Mailing List</a>.</p>
<h3>Other Projects</h3>
<p>More <a href="http://kennethreitz.org/">Kenneth Reitz</a> projects:</p>
<ul>
<li><a href="http://edmsynths.com/">edmsynths.com</a></li>
<li><a href="http://pipenv.org/">pipenv</a></li>
<li><a href="http://pep8.org/">pep8.org</a></li>
<li><a href="http://httpbin.org/">httpbin.org</a></li>
docs/_templates/sidebarlogo.html
Vendored
+11
View File
@@ -21,6 +21,15 @@
<hr/>
<h3>The Hitchhiker's Guide to Python</h3>
<p>This guide is now available in tangible book form!</p>
<a href="https://www.amazon.com/Hitchhikers-Guide-Python-Practices-Development/dp/1491933178/ref=as_li_ss_il?ie=UTF8&linkCode=li2&tag=bookforkind-20&linkId=804806ebdacaf3b56567347f3afbdbca" target="_blank"><img border="0" src="https://ws-na.amazon-adsystem.com/widgets/q?_encoding=UTF8&ASIN=1491933178&Format=_SL160_&ID=AsinImage&MarketPlace=US&ServiceVersion=20070822&WS=1&tag=bookforkind-20" ></a><img src="//ir-na.amazon-adsystem.com/e/ir?t=bookforkind-20&l=li2&o=1&a=1491933178" width="1" height="1" border="0" alt="" style="border:none !important; margin:0px !important;" />
<p>All proceeds are being directly donated to the <a href="https://djangogirls.org">DjangoGirls</a> organization.</p>
<p>If you enjoy using this project, <a href="https://saythanks.io/to/kennethreitz">Say Thanks!</a></p>
<p><iframe src="http://ghbtns.com/github-btn.html?user=kennethreitz&type=follow&count=false"
@@ -28,10 +37,12 @@
<p><a href="https://twitter.com/kennethreitz" class="twitter-follow-button" data-show-count="false">Follow @kennethreitz</a> <script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs');</script></p>
<h3>Other Projects</h3>
<p>More <a href="http://kennethreitz.org/">Kenneth Reitz</a> projects:</p>
<ul>
<li><a href="http://edmsynths.com/">edmsynths.com</a></li>
<li><a href="http://pipenv.org/">pipenv</a></li>
<li><a href="http://pep8.org/">pep8.org</a></li>
<li><a href="http://httpbin.org/">httpbin.org</a></li>
requests/sessions.py
+37 -29
View File
@@ -87,6 +87,15 @@ def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict):
class SessionRedirectMixin(object):
def get_redirect_target(self, resp):
"""Receives a Response. Returns a redirect URI or ``None``"""
if resp.is_redirect:
if not is_valid_location(response):
raise InvalidHeader('Response contains multiple Location headers. '
'Unable to perform redirect.')
return resp.headers['location']
return None
def resolve_redirects(self, response, request, stream=False, timeout=None,
verify=True, cert=None, proxies=None, **adapter_kwargs):
"""Given a Response, yields Responses until 'Location' header-based
@@ -97,61 +106,52 @@ class SessionRedirectMixin(object):
redirect_count = 0
history = [] # keep track of history
while response.is_redirect:
if not is_valid_location(response):
raise InvalidHeader('Response contains multiple Location headers. '
'Unable to perform redirect.')
url = self.get_redirect_target(response)
while url:
prepared_request = request.copy()
if redirect_count > 0:
# Store this Response in local history.
history.append(response)
# Copy local history to Response.history.
response.history = list(history)
# Update history and keep track of redirects.
# response.history must ignore the original request in this loop
hist.append(response)
response.history = hist[1:]
try:
response.content # Consume socket so it can be released
except (ChunkedEncodingError, ConnectionError, ContentDecodingError, RuntimeError):
response.raw.read(decode_content=False)
# Don't exceed configured Session.max_redirects.
if redirect_count >= self.max_redirects:
if len(response.history) >= self.max_redirects:
raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=response)
# Release the connection back into the pool.
response.close()
location_url = response.headers['location']
method = request.method
# Handle redirection without scheme (see: RFC 1808 Section 4)
if location_url.startswith('//'):
if url.startswith('//'):
parsed_rurl = urlparse(response.url)
location_url = '%s:%s' % (parsed_rurl.scheme, location_url)
location_url = '%s:%s' % (parsed_rurl.scheme, url)
# The scheme should be lower case...
parsed = urlparse(location_url)
parsed = urlparse(url)
location_url = parsed.geturl()
# On Python 3, the location header was decoded using Latin 1, but
# urlparse in requote_uri will encode it with UTF-8 before quoting.
# Because of this insanity, we need to fix it up ourselves by
# sending the URL back to bytes ourselves.
if is_py3 and isinstance(location_url, str):
location_url = location_url.encode('latin1')
if is_py3 and isinstance(url, str):
url = url.encode('latin1')
# Facilitate relative 'location' headers, as allowed by RFC 7231.
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
# Compliant with RFC3986, we percent encode the url.
if not parsed.netloc:
location_url = urljoin(response.url, requote_uri(location_url))
url = urljoin(response.url, requote_uri(url))
else:
location_url = requote_uri(location_url)
url = requote_uri(url)
prepared_request.url = to_native_string(location_url)
prepared_request.url = to_native_string(url)
# Cache the url, unless it redirects to itself.
if response.is_permanent_redirect and request.url != prepared_request.url:
self.redirect_cache[request.url] = prepared_request.url
@@ -212,7 +212,8 @@ class SessionRedirectMixin(object):
extract_cookies_to_jar(self.cookies, prepared_request, response.raw)
redirect_count += 1
# extract redirect url, if any, for the next loop
url = self.get_redirect_target(response)
yield response
def rebuild_auth(self, prepared_request, response):
@@ -252,13 +253,16 @@ class SessionRedirectMixin(object):
:rtype: dict
"""
proxies = proxies if proxies is not None else {}
headers = prepared_request.headers
url = prepared_request.url
scheme = urlparse(url).scheme
new_proxies = proxies.copy() if proxies is not None else {}
new_proxies = proxies.copy()
no_proxy = proxies.get('no_proxy')
if self.trust_env and not should_bypass_proxies(url):
environ_proxies = get_environ_proxies(url)
bypass_proxy = should_bypass_proxies(url, no_proxy=no_proxy)
if self.trust_env and not bypass_proxy:
environ_proxies = get_environ_proxies(url, no_proxy=no_proxy)
proxy = environ_proxies.get(scheme, environ_proxies.get('all'))
@@ -696,10 +700,14 @@ class Session(SessionRedirectMixin):
# can delete proxy information, which can then be re-added by a more
# specific layer. So we begin by getting the environment's proxies,
# then add the Session, then add the request.
no_proxy = proxies.get('no_proxy') if proxies is not None else None
if no_proxy is None:
no_proxy = self.proxies.get('no_proxy')
env_proxies = {}
if self.trust_env:
env_proxies = get_environ_proxies(url) or {}
env_proxies = get_environ_proxies(url, no_proxy=no_proxy) or {}
new_proxies = merge_setting(self.proxies, env_proxies)
proxies = merge_setting(proxies, new_proxies)
requests/utils.py
+43 -15
View File
@@ -11,6 +11,7 @@ that are also useful for external consumption.
import cgi
import codecs
import collections
import contextlib
import io
import os
import re
@@ -91,14 +92,16 @@ def super_len(o):
else:
if hasattr(o, 'seek') and total_length is None:
# StringIO and BytesIO have seek but no useable fileno
try:
# seek to end of file
o.seek(0, 2)
total_length = o.tell()
# seek to end of file
o.seek(0, 2)
total_length = o.tell()
# seek back to current position to support
# partially read file-like objects
o.seek(current_position or 0)
# seek back to current position to support
# partially read file-like objects
o.seek(current_position or 0)
except (OSError, IOError):
total_length = 0
if total_length is None:
total_length = 0
@@ -568,7 +571,29 @@ def is_valid_cidr(string_network):
return True
def should_bypass_proxies(url):
@contextlib.contextmanager
def set_environ(env_name, value):
"""Set the environment variable 'env_name' to 'value'
Save previous value, yield, and then restore the previous value stored in
the environment variable 'env_name'.
If 'value' is None, do nothing"""
if value is not None:
old_value = os.environ.get(env_name)
os.environ[env_name] = value
try:
yield
finally:
if value is None:
return
if old_value is None:
del os.environ[env_name]
else:
os.environ[env_name] = old_value
def should_bypass_proxies(url, no_proxy):
"""
Returns whether we should bypass proxies or not.
@@ -578,7 +603,9 @@ def should_bypass_proxies(url):
# First check whether no_proxy is defined. If it is, check that the URL
# we're getting isn't in the no_proxy list.
no_proxy = get_proxy('no_proxy')
no_proxy_arg = no_proxy
if no_proxy is None:
no_proxy = get_proxy('no_proxy')
netloc = urlparse(url).netloc
if no_proxy:
@@ -611,10 +638,11 @@ def should_bypass_proxies(url):
# of Python 2.6, so allow this call to fail. Only catch the specific
# exceptions we've seen, though: this call failing in other ways can reveal
# legitimate problems.
try:
bypass = proxy_bypass(netloc)
except (TypeError, socket.gaierror):
bypass = False
with set_environ('no_proxy', no_proxy_arg):
try:
bypass = proxy_bypass(netloc)
except (TypeError, socket.gaierror):
bypass = False
if bypass:
return True
@@ -622,13 +650,13 @@ def should_bypass_proxies(url):
return False
def get_environ_proxies(url):
def get_environ_proxies(url, no_proxy):
"""
Return a dict of environment proxies.
:rtype: dict
"""
if should_bypass_proxies(url):
if should_bypass_proxies(url, no_proxy=no_proxy):
return {}
else:
return getproxies()
tests/test_requests.py
+44 -1
View File
@@ -1650,7 +1650,7 @@ class TestRequests:
def tell(self):
return 0
def seek(self, pos):
def seek(self, pos, whence=0):
raise OSError()
def __iter__(self):
@@ -1924,6 +1924,49 @@ class TestRequests:
assert 'Transfer-Encoding' in prepared_request.headers
assert 'Content-Length' not in prepared_request.headers
def test_custom_redirect_mixin(self, httpbin):
"""Tests a custom mixin to overwrite ``get_redirect_target``.
Ensures a subclassed ``requests.Session`` can handle a certain type of
malformed redirect responses.
1. original request receives a proper response: 302 redirect
2. following the redirect, a malformed response is given:
status code = HTTP 200
location = alternate url
3. the custom session catches the edge case and follows the redirect
"""
url_final = httpbin('html')
querystring_malformed = urlencode({'location': url_final})
url_redirect_malformed = httpbin('response-headers?%s' % querystring_malformed)
querystring_redirect = urlencode({'url': url_redirect_malformed})
url_redirect = httpbin('redirect-to?%s' % querystring_redirect)
urls_test = [url_redirect,
url_redirect_malformed,
url_final,
]
class CustomRedirectSession(requests.Session):
def get_redirect_target(self, resp):
# default behavior
if resp.is_redirect:
return resp.headers['location']
# edge case - check to see if 'location' is in headers anyways
location = resp.headers.get('location')
if location and (location != resp.url):
return location
return None
session = CustomRedirectSession()
r = session.get(urls_test[0])
assert len(r.history) == 2
assert r.status_code == 200
assert r.history[0].status_code == 302
assert r.history[0].is_redirect
assert r.history[1].status_code == 200
assert not r.history[1].is_redirect
assert r.url == urls_test[2]
class TestCaseInsensitiveDict:
tests/test_utils.py
+49 -3
View File
@@ -161,7 +161,7 @@ class TestGetEnvironProxies:
'http://localhost.localdomain:5000/v1.0/',
))
def test_bypass(self, url):
assert get_environ_proxies(url) == {}
assert get_environ_proxies(url, no_proxy=None) == {}
@pytest.mark.parametrize(
'url', (
@@ -170,7 +170,32 @@ class TestGetEnvironProxies:
'http://www.requests.com/',
))
def test_not_bypass(self, url):
assert get_environ_proxies(url) != {}
assert get_environ_proxies(url, no_proxy=None) != {}
@pytest.mark.parametrize(
'url', (
'http://192.168.1.1:5000/',
'http://192.168.1.1/',
'http://www.requests.com/',
))
def test_bypass_no_proxy_keyword(self, url):
no_proxy = '192.168.1.1,requests.com'
assert get_environ_proxies(url, no_proxy=no_proxy) == {}
@pytest.mark.parametrize(
'url', (
'http://192.168.0.1:5000/',
'http://192.168.0.1/',
'http://172.16.1.1/',
'http://172.16.1.1:5000/',
'http://localhost.localdomain:5000/v1.0/',
))
def test_not_bypass_no_proxy_keyword(self, url, monkeypatch):
# This is testing that the 'no_proxy' argument overrides the
# environment variable 'no_proxy'
monkeypatch.setenv('http_proxy', 'http://proxy.example.com:3128/')
no_proxy = '192.168.1.1,requests.com'
assert get_environ_proxies(url, no_proxy=no_proxy) != {}
class TestIsIPv4Address:
@@ -525,7 +550,7 @@ def test_should_bypass_proxies(url, expected, monkeypatch):
"""
monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
assert should_bypass_proxies(url) == expected
assert should_bypass_proxies(url, no_proxy=None) == expected
@pytest.mark.parametrize(
@@ -553,3 +578,24 @@ def test_add_dict_to_cookiejar(cookiejar):
)
def test_unicode_is_ascii(value, expected):
assert unicode_is_ascii(value) is expected
@pytest.mark.parametrize(
'url, expected', (
('http://192.168.0.1:5000/', True),
('http://192.168.0.1/', True),
('http://172.16.1.1/', True),
('http://172.16.1.1:5000/', True),
('http://localhost.localdomain:5000/v1.0/', True),
('http://172.16.1.12/', False),
('http://172.16.1.12:5000/', False),
('http://google.com:5000/v1.0/', False),
))
def test_should_bypass_proxies_no_proxy(
url, expected, monkeypatch):
"""Tests for function should_bypass_proxies to check if proxy
can be bypassed or not using the 'no_proxy' argument
"""
no_proxy = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1'
# Test 'no_proxy' argument
assert should_bypass_proxies(url, no_proxy=no_proxy) == expected