From 5e86103387b39ec4b9a443e0eaacd994d04adf48 Mon Sep 17 00:00:00 2001 From: Daniele Tricoli Date: Sun, 29 May 2016 20:10:37 +0200 Subject: [PATCH 01/56] Use xfail marker for a test expected to fail without Internet connection This is only a minor improvement on the great work of https://github.com/kennethreitz/requests/pull/2859 that permits to run tests on hosts without Internet connection. --- tests/test_requests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_requests.py b/tests/test_requests.py index 167d5ca7..9e54a40f 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1107,6 +1107,7 @@ class TestRequests: preq = req.prepare() assert test_url == preq.url + @pytest.mark.xfail(raises=ConnectionError) def test_auth_is_stripped_on_redirect_off_host(self, httpbin): r = requests.get( httpbin('redirect-to'), From 8813787a12022e0a8a99ba5c8cf4c6eb6e127d36 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 4 Jun 2016 22:08:47 -0400 Subject: [PATCH 02/56] wildly inaccurate button results in deletion (not the button's fault!) --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 9fe548d2..d072d1e8 100644 --- a/README.rst +++ b/README.rst @@ -4,9 +4,6 @@ Requests: HTTP for Humans .. image:: https://img.shields.io/pypi/v/requests.svg :target: https://pypi.python.org/pypi/requests -.. image:: https://img.shields.io/pypi/dm/requests.svg - :target: https://pypi.python.org/pypi/requests - Requests is the only *Non-GMO* HTTP library for Python, safe for human consumption. From 7a404cf4ec5aac04d72dc507a981fee944b18cd8 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 8 Jun 2016 09:44:33 -0700 Subject: [PATCH 03/56] Document header ordering. (#3295) --- docs/user/advanced.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 8264e85d..cf0143ce 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -208,7 +208,7 @@ You can pass ``verify`` the path to a CA_BUNDLE file or directory with certifica >>> requests.get('https://github.com', verify='/path/to/certfile') -.. note:: If ``verify`` is set to a path to a directory, the directory must have been processed using +.. note:: If ``verify`` is set to a path to a directory, the directory must have been processed using the c_rehash utility supplied with OpenSSL. This list of trusted CAs can also be specified through the ``REQUESTS_CA_BUNDLE`` environment variable. @@ -899,6 +899,13 @@ Two excellent examples are `grequests`_ and `requests-futures`_. .. _`grequests`: https://github.com/kennethreitz/grequests .. _`requests-futures`: https://github.com/ross/requests-futures +Header Ordering +--------------- + +In unusual circumstances you may want to provide headers in an ordered manner. If you pass an ``OrderedDict`` to the ``headers`` keyword argument, that will provide the headers with an ordering. *However*, the ordering of the default headers used by requests will be preferred, which means that if you override default headers in the ``headers`` keyword argument, they may appear out of order compared to other headers in that keyword argument. + +If this is problematic, users should consider setting the default headers on a :class:`Session ` object, by setting :data:`Session ` to a custom ``OrderedDict``. That ordering will always be preferred. + .. _timeouts: Timeouts From 277d4a41ad000250bdb42322c8c238d2ff9a0df8 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Wed, 8 Jun 2016 18:19:10 -0700 Subject: [PATCH 04/56] Note how HTTPErrors are raised --- docs/user/quickstart.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index afdabe26..cc6d32df 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -492,8 +492,9 @@ Errors and Exceptions In the event of a network problem (e.g. DNS failure, refused connection, etc), Requests will raise a :class:`~requests.exceptions.ConnectionError` exception. -In the rare event of an invalid HTTP response, Requests will raise an -:class:`~requests.exceptions.HTTPError` exception. +:meth:`Response.raise_for_status() ` will +raise an :class:`~requests.exceptions.HTTPError` if the HTTP request +returned an unsuccessful status code. If a request times out, a :class:`~requests.exceptions.Timeout` exception is raised. From 6f5b6bd4c4277a3b60e412cfd561b4adbc40ca1b Mon Sep 17 00:00:00 2001 From: David Fischer Date: Wed, 8 Jun 2016 19:31:32 -0700 Subject: [PATCH 05/56] Update a note on AppEngine --- docs/dev/todo.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/dev/todo.rst b/docs/dev/todo.rst index e59213b4..59a12bf2 100644 --- a/docs/dev/todo.rst +++ b/docs/dev/todo.rst @@ -45,7 +45,10 @@ Requests currently supports the following versions of Python: Support for Python 3.1 and 3.2 may be dropped at any time. -Google App Engine will never be officially supported. Pull Requests for compatibility will be accepted, as long as they don't complicate the codebase. +Google AppEngine is not officially supported although support is available +with the `Requests-Toolbelt`_. + +.. _Requests-Toolbelt: http://toolbelt.readthedocs.io/ Are you crazy? From 6c9a0eff04e114d983d0a015ad1b0fb37da3096b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 9 Jun 2016 00:38:53 -0400 Subject: [PATCH 06/56] compensate for lack of taste --- docs/user/advanced.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index cf0143ce..b53fa436 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -193,7 +193,7 @@ SSL Cert Verification --------------------- Requests verifies SSL certificates for HTTPS requests, just like a web browser. -By default, SSL verification is enabled, and requests will throw a SSLError if +By default, SSL verification is enabled, and Requests will throw a SSLError if it's unable to verify the certificate:: >>> requests.get('https://requestb.in') @@ -235,7 +235,7 @@ If you specify a wrong path or an invalid cert, you'll get a SSLError:: SSLError: [Errno 336265225] _ssl.c:347: error:140B0009:SSL routines:SSL_CTX_use_PrivateKey_file:PEM lib .. warning:: The private key to your local certificate *must* be unencrypted. - Currently, requests does not support using encrypted keys. + Currently, Requests does not support using encrypted keys. .. _ca-certificates: @@ -552,7 +552,7 @@ SOCKS .. versionadded:: 2.10.0 -In addition to basic HTTP proxies, requests also supports proxies using the +In addition to basic HTTP proxies, Requests also supports proxies using the SOCKS protocol. This is an optional feature that requires that additional third-party libraries be installed before use. @@ -902,7 +902,7 @@ Two excellent examples are `grequests`_ and `requests-futures`_. Header Ordering --------------- -In unusual circumstances you may want to provide headers in an ordered manner. If you pass an ``OrderedDict`` to the ``headers`` keyword argument, that will provide the headers with an ordering. *However*, the ordering of the default headers used by requests will be preferred, which means that if you override default headers in the ``headers`` keyword argument, they may appear out of order compared to other headers in that keyword argument. +In unusual circumstances you may want to provide headers in an ordered manner. If you pass an ``OrderedDict`` to the ``headers`` keyword argument, that will provide the headers with an ordering. *However*, the ordering of the default headers used by Requests will be preferred, which means that if you override default headers in the ``headers`` keyword argument, they may appear out of order compared to other headers in that keyword argument. If this is problematic, users should consider setting the default headers on a :class:`Session ` object, by setting :data:`Session ` to a custom ``OrderedDict``. That ordering will always be preferred. From 400d27f982ff24784388409a70ad8945927ac023 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 17 Jun 2016 20:00:05 +0700 Subject: [PATCH 07/56] Test security warnings (#3289) Verify that the expected warnings are emitted with SubjectAltNameWarning emitted on all environments due to the https server provided by httpbin_secure. --- tests/__init__.py | 21 +++++++++++++++++++++ tests/test_requests.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index 57d631c3..3e222031 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1,22 @@ # coding: utf-8 + +"""Requests test package initialisation.""" + +import warnings + +try: + import urllib3 as urllib3_package +except ImportError: + urllib3_package = False + +from requests.packages import urllib3 as urllib3_bundle + +if urllib3_package is urllib3_bundle: + from urllib3.exceptions import SNIMissingWarning +else: + from requests.packages.urllib3.exceptions import SNIMissingWarning + +# urllib3 sets SNIMissingWarning to only go off once, +# while this test suite requires it to always fire +# so that it occurs during test_requests.test_https_warnings +warnings.simplefilter('always', SNIMissingWarning) diff --git a/tests/test_requests.py b/tests/test_requests.py index 167d5ca7..9031a9d6 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -9,6 +9,7 @@ import os import pickle import collections import contextlib +import warnings import io import requests @@ -36,6 +37,19 @@ from .utils import override_environ # listening on that port) TARPIT = 'http://10.255.255.1' +try: + from ssl import SSLContext + del SSLContext + HAS_MODERN_SSL = True +except ImportError: + HAS_MODERN_SSL = False + +try: + requests.pyopenssl + HAS_PYOPENSSL = True +except AttributeError: + HAS_PYOPENSSL = False + class TestRequests: @@ -606,6 +620,27 @@ class TestRequests: def test_pyopenssl_redirect(self, httpbin_secure, httpbin_ca_bundle): requests.get(httpbin_secure('status', '301'), verify=httpbin_ca_bundle) + def test_https_warnings(self, httpbin_secure, httpbin_ca_bundle): + """warnings are emitted with requests.get""" + if HAS_MODERN_SSL or HAS_PYOPENSSL: + warnings_expected = ('SubjectAltNameWarning', ) + else: + warnings_expected = ('SNIMissingWarning', + 'InsecurePlatformWarning', + 'SubjectAltNameWarning', ) + + with pytest.warns(None) as warning_records: + warnings.simplefilter('always') + requests.get(httpbin_secure('status', '200'), + verify=httpbin_ca_bundle) + + warning_records = [item for item in warning_records + if item.category.__name__ != 'ResourceWarning'] + + warnings_category = tuple( + item.category.__name__ for item in warning_records) + assert warnings_category == warnings_expected + def test_urlencoded_get_query_multivalued_param(self, httpbin): r = requests.get(httpbin('get'), params=dict(test=['foo', 'baz'])) From 8484144c676a09573178dc2da3f026118be48673 Mon Sep 17 00:00:00 2001 From: Peter Marsh Date: Tue, 21 Jun 2016 19:50:11 +0100 Subject: [PATCH 08/56] Update list of supported Python versions in todo.rst --- docs/dev/todo.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/dev/todo.rst b/docs/dev/todo.rst index 59a12bf2..5f1700a9 100644 --- a/docs/dev/todo.rst +++ b/docs/dev/todo.rst @@ -38,13 +38,11 @@ Requests currently supports the following versions of Python: - Python 2.6 - Python 2.7 -- Python 3.1 -- Python 3.2 - Python 3.3 +- Python 3.4 +- Python 3.5 - PyPy 1.9 -Support for Python 3.1 and 3.2 may be dropped at any time. - Google AppEngine is not officially supported although support is available with the `Requests-Toolbelt`_. From cf3c99890d73f4bc8605c1cd7fbfa4ff1a28237e Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Mon, 27 Jun 2016 18:47:34 -0400 Subject: [PATCH 09/56] added in type check for chunk_size --- requests/models.py | 2 ++ tests/test_requests.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/requests/models.py b/requests/models.py index 67747405..fbb3c7e6 100644 --- a/requests/models.py +++ b/requests/models.py @@ -685,6 +685,8 @@ class Response(object): if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() + elif not isinstance(chunk_size, int): + raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) diff --git a/tests/test_requests.py b/tests/test_requests.py index 9b614300..d2a2714a 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -980,6 +980,20 @@ class TestRequests: chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) + def test_response_chunk_size_int(self): + """Ensure that chunk_size is passed as an integer, otherwise + raise a TypeError. + """ + r = requests.Response() + r.raw = io.BytesIO(b'the content') + chunks = r.iter_content(1) + assert all(len(chunk) == 1 for chunk in chunks) + + r = requests.Response() + r.raw = io.BytesIO(b'the content') + with pytest.raises(TypeError): + chunks = r.iter_content("1024") + def test_request_and_response_are_pickleable(self, httpbin): r = requests.get(httpbin('get')) From 92fe51c0afd9239388d2c8bb17dc46babdf7881f Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Tue, 28 Jun 2016 13:22:00 -0600 Subject: [PATCH 10/56] adding asserted_encoding check on None type encoding to match text() behavior (#3362) --- requests/utils.py | 17 ++++++++++++----- tests/test_requests.py | 7 +++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 8d17b6b2..62d023fa 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -358,13 +358,20 @@ def get_encoding_from_headers(headers): def stream_decode_response_unicode(iterator, r): """Stream decodes a iterator.""" + encoding = r.encoding - if r.encoding is None: - for item in iterator: - yield item - return + if encoding is None: + encoding = r.apparent_encoding + + try: + decoder = codecs.getincrementaldecoder(encoding)(errors='replace') + except (LookupError, TypeError): + # A LookupError is raised if the encoding was not found which could + # indicate a misspelling or similar mistake. + # + # A TypeError can be raised if encoding is None + raise UnicodeError("Unable to decode contents with encoding %s." % encoding) - decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace') for chunk in iterator: rv = decoder.decode(chunk) if rv: diff --git a/tests/test_requests.py b/tests/test_requests.py index d2a2714a..4393814f 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -980,6 +980,13 @@ class TestRequests: chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) + # check for encoding value of None + r = requests.Response() + r.raw = io.BytesIO(b'the content') + r.encoding = None + chunks = r.iter_content(decode_unicode=True) + assert all(isinstance(chunk, str) for chunk in chunks) + def test_response_chunk_size_int(self): """Ensure that chunk_size is passed as an integer, otherwise raise a TypeError. From 0bcf634135e7038c2aa8b4b8b2490ad5c1db5c23 Mon Sep 17 00:00:00 2001 From: Joy Zheng Date: Thu, 30 Jun 2016 17:11:01 -0700 Subject: [PATCH 11/56] Allow None value for chunk_size again (#3368) --- requests/models.py | 2 +- tests/test_requests.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index fbb3c7e6..50b7f5da 100644 --- a/requests/models.py +++ b/requests/models.py @@ -685,7 +685,7 @@ class Response(object): if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() - elif not isinstance(chunk_size, int): + elif chunk_size is not None and not isinstance(chunk_size, int): raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) diff --git a/tests/test_requests.py b/tests/test_requests.py index 4393814f..4a16f469 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -987,8 +987,8 @@ class TestRequests: chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) - def test_response_chunk_size_int(self): - """Ensure that chunk_size is passed as an integer, otherwise + def test_response_chunk_size_type(self): + """Ensure that chunk_size is passed as None or an integer, otherwise raise a TypeError. """ r = requests.Response() @@ -996,6 +996,11 @@ class TestRequests: chunks = r.iter_content(1) assert all(len(chunk) == 1 for chunk in chunks) + r = requests.Response() + r.raw = io.BytesIO(b'the content') + chunks = r.iter_content(None) + assert list(chunks) == [b'the content'] + r = requests.Response() r.raw = io.BytesIO(b'the content') with pytest.raises(TypeError): From 0c08ca715484528b41d6957937f1d7af255d897a Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Fri, 1 Jul 2016 14:13:41 -0600 Subject: [PATCH 12/56] updating docstring to match functionality --- requests/models.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requests/models.py b/requests/models.py index 50b7f5da..7d5d8b5e 100644 --- a/requests/models.py +++ b/requests/models.py @@ -657,6 +657,12 @@ class Response(object): read into memory. This is not necessarily the length of each item returned as decoding can take place. + chunk_size must be of type int or None. A value of None will + function differently depending on the value of `stream`. + stream=True will read data as it arrives in whatever size the + chunks are recieved. If stream=False, data is returned as + a single chunk. + If decode_unicode is True, content will be decoded using the best available encoding based on the response. """ From 2669ab797ce769ecedf5493b04cb976f33e37210 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Wed, 29 Jun 2016 13:46:40 -0400 Subject: [PATCH 13/56] check and test for headers containing return characters or leading whitespace --- AUTHORS.rst | 1 + requests/exceptions.py | 6 +++++- requests/models.py | 12 ++++++++---- requests/utils.py | 20 +++++++++++++++++++- tests/test_requests.py | 43 +++++++++++++++++++++++++++++++++++++++++- 5 files changed, 75 insertions(+), 7 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 37b66698..b0ddcabb 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -166,3 +166,4 @@ Patches and Suggestions - Dmitry Dygalo (`@Stranger6667 `_) - piotrjurkiewicz - Jesse Shapiro (`@haikuginger `_) +- Nate Prewitt (`@nateprewitt `_) diff --git a/requests/exceptions.py b/requests/exceptions.py index ba0b910e..3f056492 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -80,7 +80,11 @@ class InvalidSchema(RequestException, ValueError): class InvalidURL(RequestException, ValueError): - """ The URL provided was somehow invalid. """ + """The URL provided was somehow invalid.""" + + +class InvalidHeader(RequestException, ValueError): + """The header value provided was somehow invalid.""" class ChunkedEncodingError(RequestException): diff --git a/requests/models.py b/requests/models.py index fbb3c7e6..369f790f 100644 --- a/requests/models.py +++ b/requests/models.py @@ -27,7 +27,8 @@ from .exceptions import ( from .utils import ( guess_filename, get_auth_from_url, requote_uri, stream_decode_response_unicode, to_key_val_list, parse_header_links, - iter_slices, guess_json_utf, super_len, to_native_string) + iter_slices, guess_json_utf, super_len, to_native_string, + check_header_validity) from .compat import ( cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO, is_py2, chardet, builtin_str, basestring) @@ -403,10 +404,13 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_headers(self, headers): """Prepares the given HTTP headers.""" + self.headers = CaseInsensitiveDict() if headers: - self.headers = CaseInsensitiveDict((to_native_string(name), value) for name, value in headers.items()) - else: - self.headers = CaseInsensitiveDict() + for header in headers.items(): + # Raise exception on invalid header value. + check_header_validity(header) + name, value = header + self.headers[to_native_string(name)] = value def prepare_body(self, data, files, json=None): """Prepares the given HTTP body data.""" diff --git a/requests/utils.py b/requests/utils.py index 62d023fa..769b4012 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -27,7 +27,7 @@ from .compat import (quote, urlparse, bytes, str, OrderedDict, unquote, is_py2, basestring) from .cookies import RequestsCookieJar, cookiejar_from_dict from .structures import CaseInsensitiveDict -from .exceptions import InvalidURL, FileModeWarning +from .exceptions import InvalidURL, InvalidHeader, FileModeWarning _hush_pyflakes = (RequestsCookieJar,) @@ -732,6 +732,24 @@ def to_native_string(string, encoding='ascii'): return out +# Moved outside of function to avoid recompile every call +_CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$') +_CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$') + +def check_header_validity(header): + """Verifies that header value doesn't contain leading whitespace or + return characters. This prevents unintended header injection. + + :param header: tuple, in the format (name, value). + """ + name, value = header + + if isinstance(value, bytes): + pat = _CLEAN_HEADER_REGEX_BYTE + else: + pat = _CLEAN_HEADER_REGEX_STR + if not pat.match(value): + raise InvalidHeader("Invalid return character or leading space in header: %s" % name) def urldefragauth(url): """ diff --git a/tests/test_requests.py b/tests/test_requests.py index 4393814f..5abe00aa 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -23,7 +23,7 @@ from requests.cookies import cookiejar_from_dict, morsel_to_cookie from requests.exceptions import ( ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL, MissingSchema, ReadTimeout, Timeout, RetryError, TooManyRedirects, - ProxyError) + ProxyError, InvalidHeader) from requests.models import PreparedRequest from requests.structures import CaseInsensitiveDict from requests.sessions import SessionRedirectMixin @@ -1128,6 +1128,47 @@ class TestRequests: assert 'unicode' in p.headers.keys() assert 'byte' in p.headers.keys() + def test_header_validation(self,httpbin): + """Ensure prepare_headers regex isn't flagging valid header contents.""" + headers_ok = {'foo': 'bar baz qux', + 'bar': '1', + 'baz': '', + 'qux': str.encode(u'fbbq')} + r = requests.get(httpbin('get'), headers=headers_ok) + assert r.request.headers['foo'] == headers_ok['foo'] + + def test_header_no_return_chars(self, httpbin): + """Ensure that a header containing return character sequences raise an + exception. Otherwise, multiple headers are created from single string. + """ + headers_ret = {'foo': 'bar\r\nbaz: qux'} + headers_lf = {'foo': 'bar\nbaz: qux'} + headers_cr = {'foo': 'bar\rbaz: qux'} + + # Test for newline + with pytest.raises(InvalidHeader): + r = requests.get(httpbin('get'), headers=headers_ret) + # Test for line feed + with pytest.raises(InvalidHeader): + r = requests.get(httpbin('get'), headers=headers_lf) + # Test for carriage return + with pytest.raises(InvalidHeader): + r = requests.get(httpbin('get'), headers=headers_cr) + + def test_header_no_leading_space(self, httpbin): + """Ensure headers containing leading whitespace raise + InvalidHeader Error before sending. + """ + headers_space = {'foo': ' bar'} + headers_tab = {'foo': ' bar'} + + # Test for whitespace + with pytest.raises(InvalidHeader): + r = requests.get(httpbin('get'), headers=headers_space) + # Test for tab + with pytest.raises(InvalidHeader): + r = requests.get(httpbin('get'), headers=headers_tab) + @pytest.mark.parametrize('files', ('foo', b'foo', bytearray(b'foo'))) def test_can_send_objects_with_files(self, httpbin, files): data = {'a': 'this is a string'} From 71050e9ab97dda08ca32b2b4754b6d145680c48e Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Sat, 2 Jul 2016 14:56:20 -0600 Subject: [PATCH 14/56] adding in slice_length fix and test for chunk_size=None (#3370) --- requests/utils.py | 2 ++ tests/test_utils.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 769b4012..dea323ef 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -384,6 +384,8 @@ def stream_decode_response_unicode(iterator, r): def iter_slices(string, slice_length): """Iterate over slices of a string.""" pos = 0 + if slice_length is None or slice_length <= 0: + slice_length = len(string) while pos < len(string): yield string[pos:pos + slice_length] pos += slice_length diff --git a/tests/test_utils.py b/tests/test_utils.py index 17149d26..ab5c2e37 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -378,10 +378,16 @@ def test_get_encoding_from_headers(value, expected): ('', 0), ('T', 1), ('Test', 4), + ('Cont', 0), + ('Other', -5), + ('Content', None), )) def test_iter_slices(value, length): - assert len(list(iter_slices(value, 1))) == length - + if length is None or (length <= 0 and len(value) > 0): + # Reads all content at once + assert len(list(iter_slices(value, length))) == 1 + else: + assert len(list(iter_slices(value, 1))) == length @pytest.mark.parametrize( 'value, expected', ( From 7700ecae14930fd078e28e35425661d46778bfa9 Mon Sep 17 00:00:00 2001 From: Andrii Kostenko Date: Tue, 5 Jul 2016 17:01:19 +0300 Subject: [PATCH 15/56] Support responses like `HTTP/1.1 404 Unicode chars` (#3385) --- requests/models.py | 8 ++++++-- tests/test_requests.py | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 010a1a58..d9bcfc82 100644 --- a/requests/models.py +++ b/requests/models.py @@ -849,12 +849,16 @@ class Response(object): """Raises stored :class:`HTTPError`, if one occurred.""" http_error_msg = '' + if isinstance(self.reason, bytes): + reason = self.reason.decode('utf-8', 'ignore') + else: + reason = self.reason if 400 <= self.status_code < 500: - http_error_msg = '%s Client Error: %s for url: %s' % (self.status_code, self.reason, self.url) + http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url) elif 500 <= self.status_code < 600: - http_error_msg = '%s Server Error: %s for url: %s' % (self.status_code, self.reason, self.url) + http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url) if http_error_msg: raise HTTPError(http_error_msg, response=self) diff --git a/tests/test_requests.py b/tests/test_requests.py index 4cff4da0..4250a8f9 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -987,6 +987,15 @@ class TestRequests: chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) + def test_response_reason_unicode(self): + # check for unicode HTTP status + r = requests.Response() + r.url = u'unicode URL' + r.reason = u'Komponenttia ei löydy'.encode('utf-8') + r.status_code = 404 + r.encoding = None + assert not r.ok # old behaviour - crashes here + def test_response_chunk_size_type(self): """Ensure that chunk_size is passed as None or an integer, otherwise raise a TypeError. From be31a90906deb5553c2e703fb05cf6964ee23ed5 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Tue, 5 Jul 2016 11:41:56 -0400 Subject: [PATCH 16/56] Defining header value type requirements and tests --- requests/utils.py | 13 +++++++++---- tests/test_requests.py | 24 +++++++++++++++++++++--- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index dea323ef..397a655e 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -739,8 +739,9 @@ _CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$') _CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$') def check_header_validity(header): - """Verifies that header value doesn't contain leading whitespace or - return characters. This prevents unintended header injection. + """Verifies that header value is a string which doesn't contain + leading whitespace or return characters. This prevents unintended + header injection. :param header: tuple, in the format (name, value). """ @@ -750,8 +751,12 @@ def check_header_validity(header): pat = _CLEAN_HEADER_REGEX_BYTE else: pat = _CLEAN_HEADER_REGEX_STR - if not pat.match(value): - raise InvalidHeader("Invalid return character or leading space in header: %s" % name) + try: + if not pat.match(value): + raise InvalidHeader("Invalid return character or leading space in header: %s" % name) + except TypeError: + raise InvalidHeader("Header value %s must be of type str or bytes, " + "not %s" % (value, type(value))) def urldefragauth(url): """ diff --git a/tests/test_requests.py b/tests/test_requests.py index 4250a8f9..a7d3a75b 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1142,15 +1142,33 @@ class TestRequests: assert 'unicode' in p.headers.keys() assert 'byte' in p.headers.keys() - def test_header_validation(self,httpbin): + def test_header_validation(self, httpbin): """Ensure prepare_headers regex isn't flagging valid header contents.""" headers_ok = {'foo': 'bar baz qux', - 'bar': '1', + 'bar': u'fbbq'.encode('utf8'), 'baz': '', - 'qux': str.encode(u'fbbq')} + 'qux': '1'} r = requests.get(httpbin('get'), headers=headers_ok) assert r.request.headers['foo'] == headers_ok['foo'] + def test_header_value_not_str(self, httpbin): + """Ensure the header value is of type string or bytes as + per discussion in GH issue #3386 + """ + headers_int = {'foo': 3} + headers_dict = {'bar': {'foo':'bar'}} + headers_list = {'baz': ['foo', 'bar']} + + # Test for int + with pytest.raises(InvalidHeader): + r = requests.get(httpbin('get'), headers=headers_int) + # Test for dict + with pytest.raises(InvalidHeader): + r = requests.get(httpbin('get'), headers=headers_dict) + # Test for list + with pytest.raises(InvalidHeader): + r = requests.get(httpbin('get'), headers=headers_list) + def test_header_no_return_chars(self, httpbin): """Ensure that a header containing return character sequences raise an exception. Otherwise, multiple headers are created from single string. From 1cb3b797fe1d762322eb1ab3463a615cadc51ee3 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Wed, 6 Jul 2016 15:10:46 -0600 Subject: [PATCH 17/56] updating documentation to reflect decision of #3386 --- docs/api.rst | 6 +++++- docs/user/quickstart.rst | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index 59b05232..08e2b6ee 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -258,6 +258,10 @@ Behavioural Changes * Keys in the ``headers`` dictionary are now native strings on all Python versions, i.e. bytestrings on Python 2 and unicode on Python 3. If the - keys are not native strings (unicode on Python2 or bytestrings on Python 3) + keys are not native strings (unicode on Python 2 or bytestrings on Python 3) they will be converted to the native string type assuming UTF-8 encoding. +* Values in the ``headers`` dictionary should always be strings. This has + been the project's position since before 1.0 but a recent change + (since version 2.11.0) enforces this more strictly. It's advised to avoid + passing header values as unicode when possible. diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index cc6d32df..e48a48a5 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -211,6 +211,7 @@ Note: Custom headers are given less precedence than more specific sources of inf Furthermore, Requests does not change its behavior at all based on which custom headers are specified. The headers are simply passed on into the final request. +Note: All header values must be a ``string``, bytestring, or unicode. While permitted, it's advised to avoid passing unicode header values. More complicated POST requests ------------------------------ From cf938420f88d8d534ef8b717b30da724c1e31091 Mon Sep 17 00:00:00 2001 From: Zeusw Date: Thu, 14 Jul 2016 10:40:32 +0800 Subject: [PATCH 18/56] in python3.x not have StringIO I think, should such an amendment. --- docs/user/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index e48a48a5..3be3489f 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -132,7 +132,7 @@ For example, to create an image from binary data returned by a request, you can use the following code:: >>> from PIL import Image - >>> from StringIO import StringIO + >>> from io import StringIO >>> i = Image.open(StringIO(r.content)) From 5a3aeedd1824e38ca2fe66752d54dcf22bd3785d Mon Sep 17 00:00:00 2001 From: Harrison Jackson Date: Thu, 14 Jul 2016 09:47:27 -0600 Subject: [PATCH 19/56] Remove duplicate import warnings --- requests/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/requests/__init__.py b/requests/__init__.py index 82c0f780..9a66ee1b 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -83,7 +83,5 @@ except ImportError: logging.getLogger(__name__).addHandler(NullHandler()) -import warnings - # FileModeWarnings go off per the default. warnings.simplefilter('default', FileModeWarning, append=True) From d262df99bfe0972300ca025e58a2ce1a329e3a22 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Thu, 14 Jul 2016 21:53:10 -0600 Subject: [PATCH 20/56] removing redundant test --- tests/test_requests.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_requests.py b/tests/test_requests.py index a7d3a75b..73616272 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -221,10 +221,6 @@ class TestRequests: assert r.history[0].status_code == 303 assert r.history[0].is_redirect - # def test_HTTP_302_ALLOW_REDIRECT_POST(self): - # r = requests.post(httpbin('status', '302'), data={'some': 'data'}) - # self.assertEqual(r.status_code, 200) - def test_HTTP_200_OK_GET_WITH_PARAMS(self, httpbin): heads = {'User-agent': 'Mozilla/5.0'} From 15a3869006fa68d7ad34bc315842f2ca5b537311 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Tue, 19 Jul 2016 14:51:14 -0600 Subject: [PATCH 21/56] making module docstrings and coding comments consistent --- requests/__init__.py | 1 - requests/api.py | 1 - requests/certs.py | 4 ++-- requests/compat.py | 6 +++++- requests/cookies.py | 3 +++ requests/exceptions.py | 1 - requests/hooks.py | 1 - requests/sessions.py | 1 - requests/structures.py | 1 - requests/utils.py | 1 - tests/__init__.py | 2 +- tests/compat.py | 3 ++- tests/conftest.py | 3 ++- tests/test_hooks.py | 3 ++- tests/test_lowlevel.py | 2 ++ tests/test_structures.py | 3 ++- tests/test_testserver.py | 2 ++ tests/test_utils.py | 3 ++- tests/testserver/server.py | 2 ++ tests/utils.py | 2 ++ 20 files changed, 29 insertions(+), 16 deletions(-) diff --git a/requests/__init__.py b/requests/__init__.py index 9a66ee1b..aea1fbf2 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -38,7 +38,6 @@ is at . :copyright: (c) 2016 by Kenneth Reitz. :license: Apache 2.0, see LICENSE for more details. - """ __title__ = 'requests' diff --git a/requests/api.py b/requests/api.py index c2068d0e..580b3f35 100644 --- a/requests/api.py +++ b/requests/api.py @@ -8,7 +8,6 @@ This module implements the Requests API. :copyright: (c) 2012 by Kenneth Reitz. :license: Apache2, see LICENSE for more details. - """ from . import sessions diff --git a/requests/certs.py b/requests/certs.py index 07e64750..f922b99d 100644 --- a/requests/certs.py +++ b/requests/certs.py @@ -2,8 +2,8 @@ # -*- coding: utf-8 -*- """ -certs.py -~~~~~~~~ +requests.certs +~~~~~~~~~~~~~~ This module returns the preferred default CA certificate bundle. diff --git a/requests/compat.py b/requests/compat.py index 70edff78..eb6530d6 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -1,7 +1,11 @@ # -*- coding: utf-8 -*- """ -pythoncompat +requests.compat +~~~~~~~~~~~~~~~ + +This module handles import compatibility issues between Python 2 and +Python 3. """ from .packages import chardet diff --git a/requests/cookies.py b/requests/cookies.py index eee5168f..255d92db 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- """ +requests.cookies +~~~~~~~~~~~~~~~~ + Compatibility code to be able to use `cookielib.CookieJar` with requests. requests.utils imports from here, so be careful with imports. diff --git a/requests/exceptions.py b/requests/exceptions.py index 3f056492..91de8bbc 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -5,7 +5,6 @@ requests.exceptions ~~~~~~~~~~~~~~~~~~~ This module contains the set of Requests' exceptions. - """ from .packages.urllib3.exceptions import HTTPError as BaseHTTPError diff --git a/requests/hooks.py b/requests/hooks.py index 9da94366..70d83a4b 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -10,7 +10,6 @@ Available hooks: ``response``: The response generated from a Request. - """ HOOKS = ['response'] diff --git a/requests/sessions.py b/requests/sessions.py index 3f405ba9..d2bae11f 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -6,7 +6,6 @@ requests.session This module provides a Session object to manage and persist settings across requests (cookies, auth, proxies). - """ import os from collections import Mapping diff --git a/requests/structures.py b/requests/structures.py index 991056e4..c4c78b2b 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -5,7 +5,6 @@ requests.structures ~~~~~~~~~~~~~~~~~~~ Data structures that power Requests. - """ import collections diff --git a/requests/utils.py b/requests/utils.py index 397a655e..16b51b63 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -6,7 +6,6 @@ requests.utils This module provides utility functions that are used within Requests that are also useful for external consumption. - """ import cgi diff --git a/tests/__init__.py b/tests/__init__.py index 3e222031..1b7182a5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,4 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- """Requests test package initialisation.""" diff --git a/tests/compat.py b/tests/compat.py index a26bd9f4..f68e8014 100644 --- a/tests/compat.py +++ b/tests/compat.py @@ -1,4 +1,5 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- + from requests.compat import is_py3 diff --git a/tests/conftest.py b/tests/conftest.py index af20e54d..cd64a765 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- + import pytest from requests.compat import urljoin diff --git a/tests/test_hooks.py b/tests/test_hooks.py index e2b174d8..014b4391 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -1,4 +1,5 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- + import pytest from requests import hooks diff --git a/tests/test_lowlevel.py b/tests/test_lowlevel.py index f3dd1b11..f6eaa93a 100644 --- a/tests/test_lowlevel.py +++ b/tests/test_lowlevel.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import os import pytest import threading diff --git a/tests/test_structures.py b/tests/test_structures.py index 1c332bb2..623f2b1e 100644 --- a/tests/test_structures.py +++ b/tests/test_structures.py @@ -1,4 +1,5 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- + import pytest from requests.structures import CaseInsensitiveDict, LookupDict diff --git a/tests/test_testserver.py b/tests/test_testserver.py index 9a35460e..b90a9db1 100644 --- a/tests/test_testserver.py +++ b/tests/test_testserver.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import threading import socket import time diff --git a/tests/test_utils.py b/tests/test_utils.py index ab5c2e37..6f22f659 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,5 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- + from io import BytesIO import pytest diff --git a/tests/testserver/server.py b/tests/testserver/server.py index 7a92c87d..93b6522a 100644 --- a/tests/testserver/server.py +++ b/tests/testserver/server.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import threading import socket import select diff --git a/tests/utils.py b/tests/utils.py index 6cb75bfb..9b797fd4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import contextlib import os From 2d4a89f5dc561dbf062ec5c85d6f6ca32a511042 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Tue, 19 Jul 2016 15:23:11 -0600 Subject: [PATCH 22/56] adding in pep8 fixes --- requests/auth.py | 3 +++ requests/cookies.py | 1 + requests/hooks.py | 1 + requests/models.py | 13 +++++++------ requests/sessions.py | 3 +-- requests/status_codes.py | 2 +- requests/structures.py | 2 ++ requests/utils.py | 6 ++++-- tests/test_lowlevel.py | 2 +- tests/test_requests.py | 10 ++++------ tests/test_testserver.py | 24 +++++++++++++----------- tests/test_utils.py | 1 + 12 files changed, 39 insertions(+), 29 deletions(-) diff --git a/requests/auth.py b/requests/auth.py index 73f8e9da..4f09b911 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -43,6 +43,7 @@ class AuthBase(object): class HTTPBasicAuth(AuthBase): """Attaches HTTP Basic Authentication to the given Request object.""" + def __init__(self, username, password): self.username = username self.password = password @@ -63,6 +64,7 @@ class HTTPBasicAuth(AuthBase): class HTTPProxyAuth(HTTPBasicAuth): """Attaches HTTP Proxy Authentication to a given Request object.""" + def __call__(self, r): r.headers['Proxy-Authorization'] = _basic_auth_str(self.username, self.password) return r @@ -70,6 +72,7 @@ class HTTPProxyAuth(HTTPBasicAuth): class HTTPDigestAuth(AuthBase): """Attaches HTTP Digest Authentication to the given Request object.""" + def __init__(self, username, password): self.username = username self.password = password diff --git a/requests/cookies.py b/requests/cookies.py index eee5168f..af6238c3 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -178,6 +178,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): .. warning:: dictionary operations that are normally O(1) may be O(n). """ + def get(self, name, default=None, domain=None, path=None): """Dict-like get() that also supports optional domain and path args in order to resolve naming collisions from using one cookie jar over diff --git a/requests/hooks.py b/requests/hooks.py index 9da94366..3a33da68 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -14,6 +14,7 @@ Available hooks: """ HOOKS = ['response'] + def default_hooks(): return dict((event, []) for event in HOOKS) diff --git a/requests/models.py b/requests/models.py index d9bcfc82..b80ae270 100644 --- a/requests/models.py +++ b/requests/models.py @@ -27,7 +27,7 @@ from .exceptions import ( from .utils import ( guess_filename, get_auth_from_url, requote_uri, stream_decode_response_unicode, to_key_val_list, parse_header_links, - iter_slices, guess_json_utf, super_len, to_native_string, + iter_slices, guess_json_utf, super_len, to_native_string, check_header_validity) from .compat import ( cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO, @@ -38,11 +38,11 @@ from .status_codes import codes #: The set of HTTP status codes that indicate an automatically #: processable redirect. REDIRECT_STATI = ( - codes.moved, # 301 - codes.found, # 302 - codes.other, # 303 - codes.temporary_redirect, # 307 - codes.permanent_redirect, # 308 + codes.moved, # 301 + codes.found, # 302 + codes.other, # 303 + codes.temporary_redirect, # 307 + codes.permanent_redirect, # 308 ) DEFAULT_REDIRECT_LIMIT = 30 @@ -209,6 +209,7 @@ class Request(RequestHooksMixin): """ + def __init__(self, method=None, url=None, headers=None, files=None, data=None, params=None, auth=None, cookies=None, hooks=None, json=None): diff --git a/requests/sessions.py b/requests/sessions.py index 3f405ba9..96455045 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -195,7 +195,7 @@ class SessionRedirectMixin(object): if 'Authorization' in headers: # If we get redirected to a new host, we should strip out any - # authentication headers. + # authentication headers. original_parsed = urlparse(response.request.url) redirect_parsed = urlparse(url) @@ -376,7 +376,6 @@ class Session(SessionRedirectMixin): merged_cookies = merge_cookies( merge_cookies(RequestsCookieJar(), self.cookies), cookies) - # Set environment's basic authentication if not explicitly set. auth = request.auth if self.trust_env and not auth and not self.auth: diff --git a/requests/status_codes.py b/requests/status_codes.py index 0137c91d..db2986bb 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -31,7 +31,7 @@ _codes = { 306: ('switch_proxy',), 307: ('temporary_redirect', 'temporary_moved', 'temporary'), 308: ('permanent_redirect', - 'resume_incomplete', 'resume',), # These 2 to be removed in 3.0 + 'resume_incomplete', 'resume',), # These 2 to be removed in 3.0 # Client Error. 400: ('bad_request', 'bad'), diff --git a/requests/structures.py b/requests/structures.py index 991056e4..596ac404 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -41,6 +41,7 @@ class CaseInsensitiveDict(collections.MutableMapping): behavior is undefined. """ + def __init__(self, data=None, **kwargs): self._store = OrderedDict() if data is None: @@ -87,6 +88,7 @@ class CaseInsensitiveDict(collections.MutableMapping): def __repr__(self): return str(dict(self.items())) + class LookupDict(dict): """Dictionary lookup object.""" diff --git a/requests/utils.py b/requests/utils.py index 397a655e..79b12fbe 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -384,7 +384,7 @@ def stream_decode_response_unicode(iterator, r): def iter_slices(string, slice_length): """Iterate over slices of a string.""" pos = 0 - if slice_length is None or slice_length <= 0: + if slice_length is None or slice_length <= 0: slice_length = len(string) while pos < len(string): yield string[pos:pos + slice_length] @@ -734,6 +734,7 @@ def to_native_string(string, encoding='ascii'): return out + # Moved outside of function to avoid recompile every call _CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$') _CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$') @@ -755,9 +756,10 @@ def check_header_validity(header): if not pat.match(value): raise InvalidHeader("Invalid return character or leading space in header: %s" % name) except TypeError: - raise InvalidHeader("Header value %s must be of type str or bytes, " + raise InvalidHeader("Header value %s must be of type str or bytes, " "not %s" % (value, type(value))) + def urldefragauth(url): """ Given a url remove the fragment and the authentication part diff --git a/tests/test_lowlevel.py b/tests/test_lowlevel.py index f3dd1b11..98834271 100644 --- a/tests/test_lowlevel.py +++ b/tests/test_lowlevel.py @@ -17,7 +17,7 @@ def test_chunked_upload(): with server as (host, port): url = 'http://{0}:{1}/'.format(host, port) r = requests.post(url, data=data, stream=True) - close_server.set() # release server block + close_server.set() # release server block assert r.status_code == 200 assert r.request.headers['Transfer-Encoding'] == 'chunked' diff --git a/tests/test_requests.py b/tests/test_requests.py index 73616272..e6cb8b0d 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -564,8 +564,7 @@ class TestRequests: assert post1.status_code == 200 with open('requirements.txt') as f: - post2 = requests.post(url, - data={'some': 'data'}, files={'some': f}) + post2 = requests.post(url, data={'some': 'data'}, files={'some': f}) assert post2.status_code == 200 post4 = requests.post(url, data='[{"some": "json"}]') @@ -940,8 +939,7 @@ class TestRequests: def test_time_elapsed_blank(self, httpbin): r = requests.get(httpbin('get')) td = r.elapsed - total_seconds = ((td.microseconds + (td.seconds + td.days * 24 * 3600) - * 10**6) / 10**6) + total_seconds = ((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6) assert total_seconds > 0.0 def test_response_is_iterable(self): @@ -990,7 +988,7 @@ class TestRequests: r.reason = u'Komponenttia ei löydy'.encode('utf-8') r.status_code = 404 r.encoding = None - assert not r.ok # old behaviour - crashes here + assert not r.ok # old behaviour - crashes here def test_response_chunk_size_type(self): """Ensure that chunk_size is passed as None or an integer, otherwise @@ -1152,7 +1150,7 @@ class TestRequests: per discussion in GH issue #3386 """ headers_int = {'foo': 3} - headers_dict = {'bar': {'foo':'bar'}} + headers_dict = {'bar': {'foo': 'bar'}} headers_list = {'baz': ['foo', 'bar']} # Test for int diff --git a/tests/test_testserver.py b/tests/test_testserver.py index 9a35460e..94a89483 100644 --- a/tests/test_testserver.py +++ b/tests/test_testserver.py @@ -6,16 +6,19 @@ import pytest import requests from tests.testserver.server import Server + class TestTestServer: + def test_basic(self): """messages are sent and received properly""" question = b"sucess?" answer = b"yeah, success" + def handler(sock): text = sock.recv(1000) - assert text == question + assert text == question sock.sendall(answer) - + with Server(handler) as (host, port): sock = socket.socket() sock.connect((host, port)) @@ -39,7 +42,7 @@ class TestTestServer: def test_text_response(self): """the text_response_server sends the given text""" server = Server.text_response_server( - "HTTP/1.1 200 OK\r\n" + + "HTTP/1.1 200 OK\r\n" + "Content-Length: 6\r\n" + "\r\nroflol" ) @@ -49,8 +52,8 @@ class TestTestServer: assert r.status_code == 200 assert r.text == u'roflol' - assert r.headers['Content-Length'] == '6' - + assert r.headers['Content-Length'] == '6' + def test_basic_response(self): """the basic response server returns an empty http response""" with Server.basic_response_server() as (host, port): @@ -69,12 +72,12 @@ class TestTestServer: sock.sendall(b'send something') time.sleep(2.5) sock.sendall(b'still alive') - block_server.set() # release server block + block_server.set() # release server block def test_multiple_requests(self): """multiple requests can be served""" requests_to_handle = 5 - + server = Server.basic_response_server(requests_to_handle=requests_to_handle) with server as (host, port): @@ -96,7 +99,7 @@ class TestTestServer: with server as address: sock1 = socket.socket() sock2 = socket.socket() - + sock1.connect(address) sock1.sendall(first_request) sock1.close() @@ -121,19 +124,18 @@ class TestTestServer: assert server.handler_results[0] == b'' - def test_request_recovery_with_bigger_timeout(self): """a biggest timeout can be specified""" server = Server.basic_response_server(request_timeout=3) data = b'bananadine' with server as address: - sock = socket.socket() + sock = socket.socket() sock.connect(address) time.sleep(1.5) sock.sendall(data) sock.close() - + assert server.handler_results[0] == data def test_server_finishes_on_error(self): diff --git a/tests/test_utils.py b/tests/test_utils.py index ab5c2e37..8eb0a34f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -389,6 +389,7 @@ def test_iter_slices(value, length): else: assert len(list(iter_slices(value, 1))) == length + @pytest.mark.parametrize( 'value, expected', ( ( From b7809acb473cf084b467b0ec0a524c50bcbe225f Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Wed, 20 Jul 2016 11:43:47 -0600 Subject: [PATCH 23/56] making class and function docstrings consistent --- requests/cookies.py | 89 ++++++++++++++++++++++++++++------------ requests/exceptions.py | 11 ++--- requests/models.py | 7 +--- requests/sessions.py | 18 ++++---- requests/structures.py | 4 +- requests/utils.py | 33 +++++++-------- tests/test_requests.py | 9 ++-- tests/test_structures.py | 8 +--- tests/test_utils.py | 14 +++---- 9 files changed, 101 insertions(+), 92 deletions(-) diff --git a/requests/cookies.py b/requests/cookies.py index 255d92db..19d63d88 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -161,7 +161,8 @@ def remove_cookie_by_name(cookiejar, name, domain=None, path=None): class CookieConflictError(RuntimeError): """There are two cookies that meet the criteria specified in the cookie jar. - Use .get and .set and include domain and path args in order to be more specific.""" + Use .get and .set and include domain and path args in order to be more specific. + """ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): @@ -186,7 +187,8 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): order to resolve naming collisions from using one cookie jar over multiple domains. - .. warning:: operation is O(n), not O(1).""" + .. warning:: operation is O(n), not O(1). + """ try: return self._find_no_duplicates(name, domain, path) except KeyError: @@ -195,7 +197,8 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): def set(self, name, value, **kwargs): """Dict-like set() that also supports optional domain and path args in order to resolve naming collisions from using one cookie jar over - multiple domains.""" + multiple domains. + """ # support client code that unsets cookies by assignment of a None value: if value is None: remove_cookie_by_name(self, name, domain=kwargs.get('domain'), path=kwargs.get('path')) @@ -210,37 +213,54 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): def iterkeys(self): """Dict-like iterkeys() that returns an iterator of names of cookies - from the jar. See itervalues() and iteritems().""" + from the jar. + + .. seealso:: itervalues() and iteritems(). + """ for cookie in iter(self): yield cookie.name def keys(self): """Dict-like keys() that returns a list of names of cookies from the - jar. See values() and items().""" + jar. + + .. seealso:: values() and items(). + """ return list(self.iterkeys()) def itervalues(self): """Dict-like itervalues() that returns an iterator of values of cookies - from the jar. See iterkeys() and iteritems().""" + from the jar. + + .. seealso:: iterkeys() and iteritems(). + """ for cookie in iter(self): yield cookie.value def values(self): """Dict-like values() that returns a list of values of cookies from the - jar. See keys() and items().""" + jar. + + .. seealso:: keys() and items(). + """ return list(self.itervalues()) def iteritems(self): """Dict-like iteritems() that returns an iterator of name-value tuples - from the jar. See iterkeys() and itervalues().""" + from the jar. + + .. seealso:: iterkeys() and itervalues(). + """ for cookie in iter(self): yield cookie.name, cookie.value def items(self): """Dict-like items() that returns a list of name-value tuples from the - jar. See keys() and values(). Allows client-code to call - ``dict(RequestsCookieJar)`` and get a vanilla python dict of key value - pairs.""" + jar. Allows client-code to call ``dict(RequestsCookieJar)`` and get a + vanilla python dict of key value pairs. + + .. seealso:: keys() and values(). + """ return list(self.iteritems()) def list_domains(self): @@ -261,7 +281,8 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): def multiple_domains(self): """Returns True if there are multiple domains in the jar. - Returns False otherwise.""" + Returns False otherwise. + """ domains = [] for cookie in iter(self): if cookie.domain is not None and cookie.domain in domains: @@ -272,7 +293,8 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): def get_dict(self, domain=None, path=None): """Takes as an argument an optional domain and path and returns a plain old Python dict of name-value pairs of cookies that meet the - requirements.""" + requirements. + """ dictionary = {} for cookie in iter(self): if (domain is None or cookie.domain == domain) and (path is None @@ -291,20 +313,21 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): exception if there are more than one cookie with name. In that case, use the more explicit get() method instead. - .. warning:: operation is O(n), not O(1).""" - + .. warning:: operation is O(n), not O(1). + """ return self._find_no_duplicates(name) def __setitem__(self, name, value): """Dict-like __setitem__ for compatibility with client code. Throws exception if there is already a cookie of that name in the jar. In that - case, use the more explicit set() method instead.""" - + case, use the more explicit set() method instead. + """ self.set(name, value) def __delitem__(self, name): """Deletes a cookie given a name. Wraps ``cookielib.CookieJar``'s - ``remove_cookie_by_name()``.""" + ``remove_cookie_by_name()``. + """ remove_cookie_by_name(self, name) def set_cookie(self, cookie, *args, **kwargs): @@ -321,11 +344,17 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): super(RequestsCookieJar, self).update(other) def _find(self, name, domain=None, path=None): - """Requests uses this method internally to get cookie values. Takes as - args name and optional domain and path. Returns a cookie.value. If - there are conflicting cookies, _find arbitrarily chooses one. See - _find_no_duplicates if you want an exception thrown if there are - conflicting cookies.""" + """Requests uses this method internally to get cookie values. + + If there are conflicting cookies, _find arbitrarily chooses one. + See _find_no_duplicates if you want an exception thrown if there are + conflicting cookies. + + :param name: a string containing name of cookie + :param domain: (optional) string containing domain of cookie + :param path: (optional) string containing path of cookie + :return: cookie.value + """ for cookie in iter(self): if cookie.name == name: if domain is None or cookie.domain == domain: @@ -336,10 +365,16 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): def _find_no_duplicates(self, name, domain=None, path=None): """Both ``__get_item__`` and ``get`` call this function: it's never - used elsewhere in Requests. Takes as args name and optional domain and - path. Returns a cookie.value. Throws KeyError if cookie is not found - and CookieConflictError if there are multiple cookies that match name - and optionally domain and path.""" + used elsewhere in Requests. + + :param name: a string containing name of cookie + :param domain: (optional) string containing domain of cookie + :param path: (optional) string containing path of cookie + :raises KeyError: if cookie is not found + :raises CookieConflictError: if there are multiple cookies + that match name and optionally domain and path + :return: cookie.value + """ toReturn = None for cookie in iter(self): if cookie.name == name: diff --git a/requests/exceptions.py b/requests/exceptions.py index 91de8bbc..b89e0cc6 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -11,12 +11,11 @@ from .packages.urllib3.exceptions import HTTPError as BaseHTTPError class RequestException(IOError): """There was an ambiguous exception that occurred while handling your - request.""" + request. + """ def __init__(self, *args, **kwargs): - """ - Initialize RequestException with `request` and `response` objects. - """ + """Initialize RequestException with `request` and `response` objects.""" response = kwargs.pop('response', None) self.response = response self.request = kwargs.pop('request', None) @@ -111,7 +110,5 @@ class RequestsWarning(Warning): class FileModeWarning(RequestsWarning, DeprecationWarning): - """ - A file was opened in text mode, but Requests determined its binary length. - """ + """A file was opened in text mode, but Requests determined its binary length.""" pass diff --git a/requests/models.py b/requests/models.py index d9bcfc82..e372e03b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -108,7 +108,6 @@ class RequestEncodingMixin(object): if parameters are supplied as a dict. The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype) or 4-tuples (filename, fileobj, contentype, custom_headers). - """ if (not files): raise ValueError("Files must be provided.") @@ -207,7 +206,6 @@ class Request(RequestHooksMixin): >>> req = requests.Request('GET', 'http://httpbin.org/get') >>> req.prepare() - """ def __init__(self, method=None, url=None, headers=None, files=None, data=None, params=None, auth=None, cookies=None, hooks=None, json=None): @@ -270,7 +268,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): >>> s = requests.Session() >>> s.send(r) - """ def __init__(self): @@ -516,8 +513,8 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): can only be called once for the life of the :class:`PreparedRequest ` object. Any subsequent calls to ``prepare_cookies`` will have no actual effect, unless the "Cookie" - header is removed beforehand.""" - + header is removed beforehand. + """ if isinstance(cookies, cookielib.CookieJar): self._cookies = cookies else: diff --git a/requests/sessions.py b/requests/sessions.py index d2bae11f..119eff48 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -39,9 +39,8 @@ REDIRECT_CACHE_SIZE = 1000 def merge_setting(request_setting, session_setting, dict_class=OrderedDict): - """ - Determines appropriate setting for a given request, taking into account the - explicit setting on that request, and the setting in the session. If a + """Determines appropriate setting for a given request, taking into account + the explicit setting on that request, and the setting in the session. If a setting is a dictionary, they will be merged together using `dict_class` """ @@ -71,8 +70,7 @@ def merge_setting(request_setting, session_setting, dict_class=OrderedDict): def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict): - """ - Properly merges both requests and session hooks. + """Properly merges both requests and session hooks. This is necessary because when request_hooks == {'response': []}, the merge breaks Session hooks entirely. @@ -184,8 +182,7 @@ class SessionRedirectMixin(object): yield resp def rebuild_auth(self, prepared_request, response): - """ - When being redirected we may want to strip authentication from the + """When being redirected we may want to strip authentication from the request to avoid leaking credentials. This method intelligently removes and reapplies authentication where possible to avoid credential loss. """ @@ -209,8 +206,7 @@ class SessionRedirectMixin(object): return def rebuild_proxies(self, prepared_request, proxies): - """ - This method re-evaluates the proxy configuration by considering the + """This method re-evaluates the proxy configuration by considering the environment variables. If we are redirected to a URL covered by NO_PROXY, we strip the proxy configuration. Otherwise, we set missing proxy keys for this URL (in case they were stripped by a previous @@ -661,8 +657,8 @@ class Session(SessionRedirectMixin): def mount(self, prefix, adapter): """Registers a connection adapter to a prefix. - Adapters are sorted in descending order by key length.""" - + Adapters are sorted in descending order by key length. + """ self.adapters[prefix] = adapter keys_to_move = [k for k in self.adapters if len(k) < len(prefix)] diff --git a/requests/structures.py b/requests/structures.py index c4c78b2b..85231080 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -13,8 +13,7 @@ from .compat import OrderedDict class CaseInsensitiveDict(collections.MutableMapping): - """ - A case-insensitive ``dict``-like object. + """A case-insensitive ``dict``-like object. Implements all methods and operations of ``collections.MutableMapping`` as well as dict's ``copy``. Also @@ -38,7 +37,6 @@ class CaseInsensitiveDict(collections.MutableMapping): If the constructor, ``.update``, or equality comparison operations are given keys that have equal ``.lower()``s, the behavior is undefined. - """ def __init__(self, data=None, **kwargs): self._store = OrderedDict() diff --git a/requests/utils.py b/requests/utils.py index 16b51b63..998dd0e6 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -399,7 +399,6 @@ def get_unicode_from_response(r): 1. charset from content-type 2. fall back and replace all unicode characters - """ warnings.warn(( 'In requests 3.0, get_unicode_from_response will be removed. For ' @@ -474,8 +473,8 @@ def requote_uri(uri): def address_in_network(ip, net): - """ - This function allows you to check if on IP belongs to a network subnet + """This function allows you to check if on IP belongs to a network subnet + Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24 returns False if ip = 192.168.1.1 and net = 192.168.100.0/24 """ @@ -487,8 +486,8 @@ def address_in_network(ip, net): def dotted_netmask(mask): - """ - Converts mask from /xx format to xxx.xxx.xxx.xxx + """Converts mask from /xx format to xxx.xxx.xxx.xxx + Example: if mask is 24 function returns 255.255.255.0 """ bits = 0xffffffff ^ (1 << 32 - mask) - 1 @@ -524,9 +523,7 @@ def is_valid_cidr(string_network): def should_bypass_proxies(url): - """ - Returns whether we should bypass proxies or not. - """ + """Returns whether we should bypass proxies or not.""" get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) # First check whether no_proxy is defined. If it is, check that the URL @@ -627,7 +624,6 @@ def parse_header_links(value): """Return a dict of parsed link headers proxies. i.e. Link: ; rel=front; type="image/jpeg",; rel=back;type="image/jpeg" - """ links = [] @@ -692,7 +688,8 @@ def guess_json_utf(data): def prepend_scheme_if_needed(url, new_scheme): """Given a URL that may or may not have a scheme, prepend the given scheme. - Does not replace a present scheme with the one provided as an argument.""" + Does not replace a present scheme with the one provided as an argument. + """ scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) # urlparse is a finicky beast, and sometimes decides that there isn't a @@ -706,7 +703,8 @@ def prepend_scheme_if_needed(url, new_scheme): def get_auth_from_url(url): """Given a url with authentication components, extract them into a tuple of - username,password.""" + username,password. + """ parsed = urlparse(url) try: @@ -718,10 +716,9 @@ def get_auth_from_url(url): def to_native_string(string, encoding='ascii'): - """ - Given a string object, regardless of type, returns a representation of that - string in the native string type, encoding and decoding where necessary. - This assumes ASCII unless told otherwise. + """Given a string object, regardless of type, returns a representation of + that string in the native string type, encoding and decoding where + necessary. This assumes ASCII unless told otherwise. """ if isinstance(string, builtin_str): out = string @@ -738,7 +735,7 @@ _CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$') _CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$') def check_header_validity(header): - """Verifies that header value is a string which doesn't contain + """Verifies that header value is a string which doesn't contain leading whitespace or return characters. This prevents unintended header injection. @@ -758,9 +755,7 @@ def check_header_validity(header): "not %s" % (value, type(value))) def urldefragauth(url): - """ - Given a url remove the fragment and the authentication part - """ + """Given a url remove the fragment and the authentication part""" scheme, netloc, path, params, query, fragment = urlparse(url) # see func:`prepend_scheme_if_needed` diff --git a/tests/test_requests.py b/tests/test_requests.py index 73616272..6925db43 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -957,8 +957,7 @@ class TestRequests: io.close() def test_response_decode_unicode(self): - """ - When called with decode_unicode, Response.iter_content should always + """When called with decode_unicode, Response.iter_content should always return unicode. """ r = requests.Response() @@ -1049,9 +1048,7 @@ class TestRequests: assert r.status_code == 200 def test_fixes_1329(self, httpbin): - """ - Ensure that header updates are done case-insensitively. - """ + """Ensure that header updates are done case-insensitively.""" s = requests.Session() s.headers.update({'ACCEPT': 'BOGUS'}) s.headers.update({'accept': 'application/json'}) @@ -1580,7 +1577,7 @@ class TestTimeout: assert error_text in str(e) def test_none_timeout(self, httpbin): - """ Check that you can set None as a valid timeout value. + """Check that you can set None as a valid timeout value. To actually test this behavior, we'd want to check that setting the timeout to None actually lets the request block past the system default diff --git a/tests/test_structures.py b/tests/test_structures.py index 623f2b1e..e4d2459f 100644 --- a/tests/test_structures.py +++ b/tests/test_structures.py @@ -9,9 +9,7 @@ class TestCaseInsensitiveDict: @pytest.fixture(autouse=True) def setup(self): - """ - CaseInsensitiveDict instance with "Accept" header. - """ + """CaseInsensitiveDict instance with "Accept" header.""" self.case_insensitive_dict = CaseInsensitiveDict() self.case_insensitive_dict['Accept'] = 'application/json' @@ -55,9 +53,7 @@ class TestLookupDict: @pytest.fixture(autouse=True) def setup(self): - """ - LookupDict instance with "bad_gateway" attribute. - """ + """LookupDict instance with "bad_gateway" attribute.""" self.lookup_dict = LookupDict('test') self.lookup_dict.bad_gateway = 502 diff --git a/tests/test_utils.py b/tests/test_utils.py index 6f22f659..b78c7359 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -41,9 +41,7 @@ class TestSuperLen: @pytest.mark.parametrize('error', [IOError, OSError]) def test_super_len_handles_files_raising_weird_errors_in_tell(self, error): - """ - If tell() raises errors, assume the cursor is at position zero. - """ + """If tell() raises errors, assume the cursor is at position zero.""" class BoomFile(object): def __len__(self): return 5 @@ -105,7 +103,8 @@ class TestUnquoteHeaderValue: class TestGetEnvironProxies: """Ensures that IP addresses are correctly matches with ranges - in no_proxy variable.""" + in no_proxy variable. + """ @pytest.fixture(autouse=True, params=['no_proxy', 'NO_PROXY']) def no_proxy(self, request, monkeypatch): @@ -289,8 +288,7 @@ def test_get_auth_from_url(url, auth): ), )) def test_requote_uri_with_unquoted_percents(uri, expected): - """See: https://github.com/kennethreitz/requests/issues/2356 - """ + """See: https://github.com/kennethreitz/requests/issues/2356""" assert requote_uri(uri) == expected @@ -460,8 +458,8 @@ def test_urldefragauth(url, expected): ('http://google.com:5000/v1.0/', False), )) def test_should_bypass_proxies(url, expected, monkeypatch): - """ - Tests for function should_bypass_proxies to check if proxy can be bypassed or not + """Tests for function should_bypass_proxies to check if proxy + can be bypassed or not """ monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') From 5453b88f8fa97e5ff0b7e093802eb6a8b2d90a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Tue, 26 Jul 2016 15:24:29 +0300 Subject: [PATCH 24/56] Spelling fixes --- requests/models.py | 2 +- tests/test_testserver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index adbcf3c0..ba70e3a6 100644 --- a/requests/models.py +++ b/requests/models.py @@ -662,7 +662,7 @@ class Response(object): chunk_size must be of type int or None. A value of None will function differently depending on the value of `stream`. stream=True will read data as it arrives in whatever size the - chunks are recieved. If stream=False, data is returned as + chunks are received. If stream=False, data is returned as a single chunk. If decode_unicode is True, content will be decoded using the best diff --git a/tests/test_testserver.py b/tests/test_testserver.py index b7c3fed7..0998d9a4 100644 --- a/tests/test_testserver.py +++ b/tests/test_testserver.py @@ -13,7 +13,7 @@ class TestTestServer: def test_basic(self): """messages are sent and received properly""" - question = b"sucess?" + question = b"success?" answer = b"yeah, success" def handler(sock): From c69e3eed312404e469dae250592ea6c5b20ebc64 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 30 Jul 2016 07:16:18 -0500 Subject: [PATCH 25/56] Close and then release the connection urllib3 closes the underlying connection when we call urllib3.Response.close but does not release it back to the connection pool. This can cause issues when users have a blocking connection pool configured and connections are not readily returned to the pool. Since the underlying connection is closed, we should be able to safely return the connection to the connection pool, so to fix this issue we merely need to not return after closing the response. Closes gh-3461 --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index ba70e3a6..11434ef4 100644 --- a/requests/models.py +++ b/requests/models.py @@ -868,6 +868,6 @@ class Response(object): *Note: Should not normally need to be called explicitly.* """ if not self._content_consumed: - return self.raw.close() + self.raw.close() return self.raw.release_conn() From 07b7872df8df67eaf61e6374c39767219e9155d7 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 3 Aug 2016 13:03:34 -0700 Subject: [PATCH 26/56] Add a bit about RequestsCookieJar to Cookies section --- docs/user/quickstart.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 3be3489f..ea87eecc 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -416,6 +416,19 @@ parameter:: >>> r = requests.get(url, cookies=cookies) >>> r.text '{"cookies": {"cookies_are": "working"}}' + +Cookies are returned in a :class:`~requests.cookies.RequestsCookieJar`, +which acts like a ``dict`` but also offers a more complete interface, +suitable for use over multiple domains or paths. Cookie jars can +also be passed in to requests:: + + >>> jar = requests.cookies.RequestsCookieJar() + >>> jar.set('tasty_cookie', 'yum', site='httpbin.org', path='/cookies') + >>> jar.set('gross_cookie', 'blech', site='httpbin.org', path='/elsewhere') + >>> url = 'http://httpbin.org/cookies' + >>> r = requests.get(url, cookies=jar) + >>> r.text + '{"cookies": {"tasty_cookie": "yum"}}' Redirection and History From 7bf67b4101324a729d640818803582fb892520d4 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 8 Aug 2016 13:21:06 +0100 Subject: [PATCH 27/56] Update urllib3 to 1.16 --- requests/packages/urllib3/__init__.py | 2 +- requests/packages/urllib3/connectionpool.py | 37 +- .../packages/urllib3/contrib/appengine.py | 2 +- requests/packages/urllib3/contrib/socks.py | 2 +- requests/packages/urllib3/packages/six.py | 631 ++++++++++++++++-- .../packages/ssl_match_hostname/.gitignore | 1 + requests/packages/urllib3/poolmanager.py | 93 ++- requests/packages/urllib3/response.py | 4 + requests/packages/urllib3/util/connection.py | 47 +- requests/packages/urllib3/util/retry.py | 14 +- requests/packages/urllib3/util/ssl_.py | 4 +- 11 files changed, 737 insertions(+), 100 deletions(-) create mode 100644 requests/packages/urllib3/packages/ssl_match_hostname/.gitignore diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 73668991..c3536742 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -32,7 +32,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.15.1' +__version__ = '1.16' __all__ = ( 'HTTPConnectionPool', diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 3fcfb120..ab634cb4 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -90,7 +90,7 @@ class ConnectionPool(object): # Return False to re-raise any potential exceptions return False - def close(): + def close(self): """ Close all pooled connections and disable the pool. """ @@ -163,6 +163,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): scheme = 'http' ConnectionCls = HTTPConnection + ResponseCls = HTTPResponse def __init__(self, host, port=None, strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False, @@ -383,8 +384,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): try: try: # Python 2.7, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) - except TypeError: # Python 2.6 and older - httplib_response = conn.getresponse() + except TypeError: # Python 2.6 and older, Python 3 + try: + httplib_response = conn.getresponse() + except Exception as e: + # Remove the TypeError from the exception chain in Python 3; + # otherwise it looks like a programming error was the cause. + six.raise_from(e, None) except (SocketTimeout, BaseSSLError, SocketError) as e: self._raise_timeout(err=e, url=url, timeout_value=read_timeout) raise @@ -545,6 +551,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn = None + # Track whether `conn` needs to be released before + # returning/raising/recursing. Update this variable if necessary, and + # leave `release_conn` constant throughout the function. That way, if + # the function recurses, the original value of `release_conn` will be + # passed down into the recursive call, and its value will be respected. + # + # See issue #651 [1] for details. + # + # [1] + release_this_conn = release_conn + # Merge the proxy headers. Only do this in HTTP. We have to copy the # headers dict so we can safely change it without those changes being # reflected in anyone else's copy. @@ -584,10 +601,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): response_conn = conn if not release_conn else None # Import httplib's response into our own wrapper object - response = HTTPResponse.from_httplib(httplib_response, - pool=self, - connection=response_conn, - **response_kw) + response = self.ResponseCls.from_httplib(httplib_response, + pool=self, + connection=response_conn, + **response_kw) # Everything went great! clean_exit = True @@ -633,9 +650,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Close the connection, set the variable to None, and make sure # we put the None back in the pool to avoid leaking it. conn = conn and conn.close() - release_conn = True + release_this_conn = True - if release_conn: + if release_this_conn: # Put the connection back to be reused. If the connection is # expired then it will be None, which will get replaced with a # fresh connection during _get_conn. @@ -817,7 +834,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): warnings.warn(( 'Unverified HTTPS request is being made. ' 'Adding certificate verification is strongly advised. See: ' - 'https://urllib3.readthedocs.org/en/latest/security.html'), + 'https://urllib3.readthedocs.io/en/latest/security.html'), InsecureRequestWarning) diff --git a/requests/packages/urllib3/contrib/appengine.py b/requests/packages/urllib3/contrib/appengine.py index f4289c0f..1579476c 100644 --- a/requests/packages/urllib3/contrib/appengine.py +++ b/requests/packages/urllib3/contrib/appengine.py @@ -70,7 +70,7 @@ class AppEngineManager(RequestMethods): warnings.warn( "urllib3 is using URLFetch on Google App Engine sandbox instead " "of sockets. To use sockets directly instead of URLFetch see " - "https://urllib3.readthedocs.org/en/latest/contrib.html.", + "https://urllib3.readthedocs.io/en/latest/contrib.html.", AppEnginePlatformWarning) RequestMethods.__init__(self, headers) diff --git a/requests/packages/urllib3/contrib/socks.py b/requests/packages/urllib3/contrib/socks.py index 3748fee5..81970fa6 100644 --- a/requests/packages/urllib3/contrib/socks.py +++ b/requests/packages/urllib3/contrib/socks.py @@ -26,7 +26,7 @@ except ImportError: warnings.warn(( 'SOCKS support in urllib3 requires the installation of optional ' 'dependencies: specifically, PySocks. For more information, see ' - 'https://urllib3.readthedocs.org/en/latest/contrib.html#socks-proxies' + 'https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies' ), DependencyWarning ) diff --git a/requests/packages/urllib3/packages/six.py b/requests/packages/urllib3/packages/six.py index 27d80112..190c0239 100644 --- a/requests/packages/urllib3/packages/six.py +++ b/requests/packages/urllib3/packages/six.py @@ -1,34 +1,41 @@ """Utilities for writing code that runs on Python 2 and 3""" -#Copyright (c) 2010-2011 Benjamin Peterson +# Copyright (c) 2010-2015 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. -#Permission is hereby granted, free of charge, to any person obtaining a copy of -#this software and associated documentation files (the "Software"), to deal in -#the Software without restriction, including without limitation the rights to -#use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -#the Software, and to permit persons to whom the Software is furnished to do so, -#subject to the following conditions: - -#The above copyright notice and this permission notice shall be included in all -#copies or substantial portions of the Software. - -#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -#COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -#IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -#CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +from __future__ import absolute_import +import functools +import itertools import operator import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.2.0" # Revision 41c74fef2ded +__version__ = "1.10.0" -# True if we are running on Python 3. +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) if PY3: string_types = str, @@ -51,6 +58,7 @@ else: else: # It's possible to have sizeof(long) != sizeof(Py_ssize_t). class X(object): + def __len__(self): return 1 << 31 try: @@ -61,7 +69,7 @@ else: else: # 64-bit MAXSIZE = int((1 << 63) - 1) - del X + del X def _add_doc(func, doc): @@ -82,9 +90,13 @@ class _LazyDescr(object): def __get__(self, obj, tp): result = self._resolve() - setattr(obj, self.name, result) - # This is a bit ugly, but it avoids running this again. - delattr(tp, self.name) + setattr(obj, self.name, result) # Invokes __set__. + try: + # This is a bit ugly, but it avoids running this again by + # removing this descriptor. + delattr(obj.__class__, self.name) + except AttributeError: + pass return result @@ -102,6 +114,27 @@ class MovedModule(_LazyDescr): def _resolve(self): return _import_module(self.mod) + def __getattr__(self, attr): + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + class MovedAttribute(_LazyDescr): @@ -128,30 +161,111 @@ class MovedAttribute(_LazyDescr): return getattr(module, self.attr) +class _SixMetaPathImporter(object): + + """ + A meta path importer to import six.moves and its submodules. + + This class implements a PEP302 finder and loader. It should be compatible + with Python 2.5 and all existing versions of Python3 + """ + + def __init__(self, six_module_name): + self.name = six_module_name + self.known_modules = {} + + def _add_module(self, mod, *fullnames): + for fullname in fullnames: + self.known_modules[self.name + "." + fullname] = mod + + def _get_module(self, fullname): + return self.known_modules[self.name + "." + fullname] + + def find_module(self, fullname, path=None): + if fullname in self.known_modules: + return self + return None + + def __get_module(self, fullname): + try: + return self.known_modules[fullname] + except KeyError: + raise ImportError("This loader does not know module " + fullname) + + def load_module(self, fullname): + try: + # in case of a reload + return sys.modules[fullname] + except KeyError: + pass + mod = self.__get_module(fullname) + if isinstance(mod, MovedModule): + mod = mod._resolve() + else: + mod.__loader__ = self + sys.modules[fullname] = mod + return mod + + def is_package(self, fullname): + """ + Return true, if the named module is a package. + + We need this method to get correct spec objects with + Python 3.4 (see PEP451) + """ + return hasattr(self.__get_module(fullname), "__path__") + + def get_code(self, fullname): + """Return None + + Required, if is_package is implemented""" + self.__get_module(fullname) # eventually raises ImportError + return None + get_source = get_code # same as get_code + +_importer = _SixMetaPathImporter(__name__) + + +class _MovedItems(_LazyModule): -class _MovedItems(types.ModuleType): """Lazy loading of moved objects""" + __path__ = [] # mark as package _moved_attributes = [ MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("intern", "__builtin__", "sys"), MovedAttribute("map", "itertools", "builtins", "imap", "map"), - MovedAttribute("reload_module", "__builtin__", "imp", "reload"), + MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), + MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserList", "UserList", "collections"), + MovedAttribute("UserString", "UserString", "collections"), MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), MovedModule("http_cookies", "Cookie", "http.cookies"), MovedModule("html_entities", "htmlentitydefs", "html.entities"), MovedModule("html_parser", "HTMLParser", "html.parser"), MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), @@ -159,12 +273,14 @@ _moved_attributes = [ MovedModule("queue", "Queue"), MovedModule("reprlib", "repr"), MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), MovedModule("tkinter", "Tkinter"), MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), MovedModule("tkinter_colorchooser", "tkColorChooser", @@ -176,14 +292,195 @@ _moved_attributes = [ MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), - MovedModule("winreg", "_winreg"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), ] +# Add windows specific modules. +if sys.platform == "win32": + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] + for attr in _moved_attributes: setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + _importer._add_module(attr, "moves." + attr.name) del attr -moves = sys.modules[__name__ + ".moves"] = _MovedItems("moves") +_MovedItems._moved_attributes = _moved_attributes + +moves = _MovedItems(__name__ + ".moves") +_importer._add_module(moves, "moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), + MovedAttribute("splittag", "urllib", "urllib.parse"), + MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), + MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), + MovedAttribute("uses_params", "urlparse", "urllib.parse"), + MovedAttribute("uses_query", "urlparse", "urllib.parse"), + MovedAttribute("uses_relative", "urlparse", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", "moves.urllib.parse") + + +class Module_six_moves_urllib_error(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", "moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", "moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", "moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", "moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package + parse = _importer._get_module("moves.urllib_parse") + error = _importer._get_module("moves.urllib_error") + request = _importer._get_module("moves.urllib_request") + response = _importer._get_module("moves.urllib_response") + robotparser = _importer._get_module("moves.urllib_robotparser") + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + +_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), + "moves.urllib") def add_move(move): @@ -206,22 +503,18 @@ if PY3: _meth_func = "__func__" _meth_self = "__self__" + _func_closure = "__closure__" _func_code = "__code__" _func_defaults = "__defaults__" - - _iterkeys = "keys" - _itervalues = "values" - _iteritems = "items" + _func_globals = "__globals__" else: _meth_func = "im_func" _meth_self = "im_self" + _func_closure = "func_closure" _func_code = "func_code" _func_defaults = "func_defaults" - - _iterkeys = "iterkeys" - _itervalues = "itervalues" - _iteritems = "iteritems" + _func_globals = "func_globals" try: @@ -232,18 +525,33 @@ except NameError: next = advance_iterator +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + if PY3: def get_unbound_function(unbound): return unbound - Iterator = object + create_bound_method = types.MethodType - def callable(obj): - return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + def create_unbound_method(func, cls): + return func + + Iterator = object else: def get_unbound_function(unbound): return unbound.im_func + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + def create_unbound_method(func, cls): + return types.MethodType(func, None, cls) + class Iterator(object): def next(self): @@ -256,90 +564,179 @@ _add_doc(get_unbound_function, get_method_function = operator.attrgetter(_meth_func) get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) get_function_code = operator.attrgetter(_func_code) get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) -def iterkeys(d): - """Return an iterator over the keys of a dictionary.""" - return iter(getattr(d, _iterkeys)()) +if PY3: + def iterkeys(d, **kw): + return iter(d.keys(**kw)) -def itervalues(d): - """Return an iterator over the values of a dictionary.""" - return iter(getattr(d, _itervalues)()) + def itervalues(d, **kw): + return iter(d.values(**kw)) -def iteritems(d): - """Return an iterator over the (key, value) pairs of a dictionary.""" - return iter(getattr(d, _iteritems)()) + def iteritems(d, **kw): + return iter(d.items(**kw)) + + def iterlists(d, **kw): + return iter(d.lists(**kw)) + + viewkeys = operator.methodcaller("keys") + + viewvalues = operator.methodcaller("values") + + viewitems = operator.methodcaller("items") +else: + def iterkeys(d, **kw): + return d.iterkeys(**kw) + + def itervalues(d, **kw): + return d.itervalues(**kw) + + def iteritems(d, **kw): + return d.iteritems(**kw) + + def iterlists(d, **kw): + return d.iterlists(**kw) + + viewkeys = operator.methodcaller("viewkeys") + + viewvalues = operator.methodcaller("viewvalues") + + viewitems = operator.methodcaller("viewitems") + +_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") +_add_doc(itervalues, "Return an iterator over the values of a dictionary.") +_add_doc(iteritems, + "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc(iterlists, + "Return an iterator over the (key, [values]) pairs of a dictionary.") if PY3: def b(s): return s.encode("latin-1") + def u(s): return s - if sys.version_info[1] <= 1: - def int2byte(i): - return bytes((i,)) - else: - # This is about 2x faster than the implementation above on 3.2+ - int2byte = operator.methodcaller("to_bytes", 1, "big") + unichr = chr + import struct + int2byte = struct.Struct(">B").pack + del struct + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter import io StringIO = io.StringIO BytesIO = io.BytesIO + _assertCountEqual = "assertCountEqual" + if sys.version_info[1] <= 1: + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + else: + _assertRaisesRegex = "assertRaisesRegex" + _assertRegex = "assertRegex" else: def b(s): return s + # Workaround for standalone backslash + def u(s): - return unicode(s, "unicode_escape") + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr int2byte = chr + + def byte2int(bs): + return ord(bs[0]) + + def indexbytes(buf, i): + return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) import StringIO StringIO = BytesIO = StringIO.StringIO + _assertCountEqual = "assertItemsEqual" + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" _add_doc(b, """Byte literal""") _add_doc(u, """Text literal""") -if PY3: - import builtins - exec_ = getattr(builtins, "exec") +def assertCountEqual(self, *args, **kwargs): + return getattr(self, _assertCountEqual)(*args, **kwargs) +def assertRaisesRegex(self, *args, **kwargs): + return getattr(self, _assertRaisesRegex)(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return getattr(self, _assertRegex)(*args, **kwargs) + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + def reraise(tp, value, tb=None): + if value is None: + value = tp() if value.__traceback__ is not tb: raise value.with_traceback(tb) raise value - - print_ = getattr(builtins, "print") - del builtins - else: - def exec_(code, globs=None, locs=None): + def exec_(_code_, _globs_=None, _locs_=None): """Execute code in a namespace.""" - if globs is None: + if _globs_ is None: frame = sys._getframe(1) - globs = frame.f_globals - if locs is None: - locs = frame.f_locals + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals del frame - elif locs is None: - locs = globs - exec("""exec code in globs, locs""") - + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") exec_("""def reraise(tp, value, tb=None): raise tp, value, tb """) +if sys.version_info[:2] == (3, 2): + exec_("""def raise_from(value, from_value): + if from_value is None: + raise value + raise value from from_value +""") +elif sys.version_info[:2] > (3, 2): + exec_("""def raise_from(value, from_value): + raise value from from_value +""") +else: + def raise_from(value, from_value): + raise value + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: def print_(*args, **kwargs): - """The new-style print function.""" + """The new-style print function for Python 2.4 and 2.5.""" fp = kwargs.pop("file", sys.stdout) if fp is None: return + def write(data): if not isinstance(data, basestring): data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) fp.write(data) want_unicode = False sep = kwargs.pop("sep", None) @@ -376,10 +773,96 @@ else: write(sep) write(arg) write(end) +if sys.version_info[:2] < (3, 3): + _print = print_ + + def print_(*args, **kwargs): + fp = kwargs.get("file", sys.stdout) + flush = kwargs.pop("flush", False) + _print(*args, **kwargs) + if flush and fp is not None: + fp.flush() _add_doc(reraise, """Reraise an exception.""") +if sys.version_info[0:2] < (3, 4): + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + def wrapper(f): + f = functools.wraps(wrapped, assigned, updated)(f) + f.__wrapped__ = wrapped + return f + return wrapper +else: + wraps = functools.wraps -def with_metaclass(meta, base=object): + +def with_metaclass(meta, *bases): """Create a base class with a metaclass.""" - return meta("NewBase", (base,), {}) + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper + + +def python_2_unicode_compatible(klass): + """ + A decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + if '__str__' not in klass.__dict__: + raise ValueError("@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + + +# Complete the moves implementation. +# This code is at the end of this module to speed up module loading. +# Turn this module into a package. +__path__ = [] # required for PEP 302 and PEP 451 +__package__ = __name__ # see PEP 366 @ReservedAssignment +if globals().get("__spec__") is not None: + __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable +# Remove other six meta path importers, since they cause problems. This can +# happen if six is removed from sys.modules and then reloaded. (Setuptools does +# this for some reason.) +if sys.meta_path: + for i, importer in enumerate(sys.meta_path): + # Here's some real nastiness: Another "instance" of the six module might + # be floating around. Therefore, we can't use isinstance() to check for + # the six meta path importer, since the other six instance will have + # inserted an importer with different class. + if (type(importer).__name__ == "_SixMetaPathImporter" and + importer.name == __name__): + del sys.meta_path[i] + break + del i, importer +# Finally, add the importer to the meta path import hook. +sys.meta_path.append(_importer) diff --git a/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore b/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore new file mode 100644 index 00000000..0a764a4d --- /dev/null +++ b/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore @@ -0,0 +1 @@ +env diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index 1023dcba..7ed00b1c 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -1,4 +1,6 @@ from __future__ import absolute_import +import collections +import functools import logging try: # Python 3 @@ -23,6 +25,59 @@ log = logging.getLogger(__name__) SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', 'ssl_version', 'ca_cert_dir') +# The base fields to use when determining what pool to get a connection from; +# these do not rely on the ``connection_pool_kw`` and can be determined by the +# URL and potentially the ``urllib3.connection.port_by_scheme`` dictionary. +# +# All custom key schemes should include the fields in this key at a minimum. +BasePoolKey = collections.namedtuple('BasePoolKey', ('scheme', 'host', 'port')) + +# The fields to use when determining what pool to get a HTTP and HTTPS +# connection from. All additional fields must be present in the PoolManager's +# ``connection_pool_kw`` instance variable. +HTTPPoolKey = collections.namedtuple( + 'HTTPPoolKey', BasePoolKey._fields + ('timeout', 'retries', 'strict', + 'block', 'source_address') +) +HTTPSPoolKey = collections.namedtuple( + 'HTTPSPoolKey', HTTPPoolKey._fields + SSL_KEYWORDS +) + + +def _default_key_normalizer(key_class, request_context): + """ + Create a pool key of type ``key_class`` for a request. + + According to RFC 3986, both the scheme and host are case-insensitive. + Therefore, this function normalizes both before constructing the pool + key for an HTTPS request. If you wish to change this behaviour, provide + alternate callables to ``key_fn_by_scheme``. + + :param key_class: + The class to use when constructing the key. This should be a namedtuple + with the ``scheme`` and ``host`` keys at a minimum. + + :param request_context: + A dictionary-like object that contain the context for a request. + It should contain a key for each field in the :class:`HTTPPoolKey` + """ + context = {} + for key in key_class._fields: + context[key] = request_context.get(key) + context['scheme'] = context['scheme'].lower() + context['host'] = context['host'].lower() + return key_class(**context) + + +# A dictionary that maps a scheme to a callable that creates a pool key. +# This can be used to alter the way pool keys are constructed, if desired. +# Each PoolManager makes a copy of this dictionary so they can be configured +# globally here, or individually on the instance. +key_fn_by_scheme = { + 'http': functools.partial(_default_key_normalizer, HTTPPoolKey), + 'https': functools.partial(_default_key_normalizer, HTTPSPoolKey), +} + pool_classes_by_scheme = { 'http': HTTPConnectionPool, 'https': HTTPSConnectionPool, @@ -65,8 +120,10 @@ class PoolManager(RequestMethods): self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) - # Locally set the pool classes so other PoolManagers can override them. + # Locally set the pool classes and keys so other PoolManagers can + # override them. self.pool_classes_by_scheme = pool_classes_by_scheme + self.key_fn_by_scheme = key_fn_by_scheme.copy() def __enter__(self): return self @@ -113,10 +170,36 @@ class PoolManager(RequestMethods): if not host: raise LocationValueError("No host specified.") - scheme = scheme or 'http' - port = port or port_by_scheme.get(scheme, 80) - pool_key = (scheme, host, port) + request_context = self.connection_pool_kw.copy() + request_context['scheme'] = scheme or 'http' + if not port: + port = port_by_scheme.get(request_context['scheme'].lower(), 80) + request_context['port'] = port + request_context['host'] = host + return self.connection_from_context(request_context) + + def connection_from_context(self, request_context): + """ + Get a :class:`ConnectionPool` based on the request context. + + ``request_context`` must at least contain the ``scheme`` key and its + value must be a key in ``key_fn_by_scheme`` instance variable. + """ + scheme = request_context['scheme'].lower() + pool_key_constructor = self.key_fn_by_scheme[scheme] + pool_key = pool_key_constructor(request_context) + + return self.connection_from_pool_key(pool_key) + + def connection_from_pool_key(self, pool_key): + """ + Get a :class:`ConnectionPool` based on the provided pool key. + + ``pool_key`` should be a namedtuple that only contains immutable + objects. At a minimum it must have the ``scheme``, ``host``, and + ``port`` fields. + """ with self.pools.lock: # If the scheme, host, or port doesn't match existing open # connections, open a new ConnectionPool. @@ -125,7 +208,7 @@ class PoolManager(RequestMethods): return pool # Make a fresh ConnectionPool of the desired type - pool = self._new_pool(scheme, host, port) + pool = self._new_pool(pool_key.scheme, pool_key.host, pool_key.port) self.pools[pool_key] = pool return pool diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index ac1b2f19..55679032 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -165,6 +165,10 @@ class HTTPResponse(io.IOBase): if self._fp: return self.read(cache_content=True) + @property + def connection(self): + return self._connection + def tell(self): """ Obtain the number of bytes pulled over the wire so far. May differ from diff --git a/requests/packages/urllib3/util/connection.py b/requests/packages/urllib3/util/connection.py index 01a4812f..5e761352 100644 --- a/requests/packages/urllib3/util/connection.py +++ b/requests/packages/urllib3/util/connection.py @@ -46,6 +46,8 @@ def is_connection_dropped(conn): # Platform-specific # This function is copied from socket.py in the Python 2.7 standard # library test suite. Added to its signature is only `socket_options`. +# One additional modification is that we avoid binding to IPv6 servers +# discovered in DNS if the system doesn't have IPv6 functionality. def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None, socket_options=None): """Connect to *address* and return the socket object. @@ -64,14 +66,19 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, if host.startswith('['): host = host.strip('[]') err = None - for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): + + # Using the value from allowed_gai_family() in the context of getaddrinfo lets + # us select whether to work with IPv4 DNS records, IPv6 records, or both. + # The original create_connection function always returns all records. + family = allowed_gai_family() + + for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res sock = None try: sock = socket.socket(af, socktype, proto) # If provided, set socket level options before connecting. - # This is the only addition urllib3 makes to this function. _set_socket_options(sock, socket_options) if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: @@ -99,3 +106,39 @@ def _set_socket_options(sock, options): for opt in options: sock.setsockopt(*opt) + + +def allowed_gai_family(): + """This function is designed to work in the context of + getaddrinfo, where family=socket.AF_UNSPEC is the default and + will perform a DNS search for both IPv6 and IPv4 records.""" + + family = socket.AF_INET + if HAS_IPV6: + family = socket.AF_UNSPEC + return family + + +def _has_ipv6(host): + """ Returns True if the system can bind an IPv6 address. """ + sock = None + has_ipv6 = False + + if socket.has_ipv6: + # has_ipv6 returns true if cPython was compiled with IPv6 support. + # It does not tell us if the system has IPv6 support enabled. To + # determine that we must bind to an IPv6 address. + # https://github.com/shazow/urllib3/pull/611 + # https://bugs.python.org/issue658327 + try: + sock = socket.socket(socket.AF_INET6) + sock.bind((host, 0)) + has_ipv6 = True + except Exception: + pass + + if sock: + sock.close() + return has_ipv6 + +HAS_IPV6 = _has_ipv6('::1') diff --git a/requests/packages/urllib3/util/retry.py b/requests/packages/urllib3/util/retry.py index 2d3aa20d..d379833c 100644 --- a/requests/packages/urllib3/util/retry.py +++ b/requests/packages/urllib3/util/retry.py @@ -80,21 +80,27 @@ class Retry(object): Set of uppercased HTTP method verbs that we should retry on. By default, we only retry on methods which are considered to be - indempotent (multiple requests with the same parameters end with the + idempotent (multiple requests with the same parameters end with the same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`. + Set to a ``False`` value to retry on any verb. + :param iterable status_forcelist: - A set of HTTP status codes that we should force a retry on. + A set of integer HTTP status codes that we should force a retry on. + A retry is initiated if the request method is in ``method_whitelist`` + and the response status code is in ``status_forcelist``. By default, this is disabled with ``None``. :param float backoff_factor: - A backoff factor to apply between attempts. urllib3 will sleep for:: + A backoff factor to apply between attempts after the second try + (most errors are resolved immediately by a second try without a + delay). urllib3 will sleep for:: {backoff factor} * (2 ^ ({number of total retries} - 1)) seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep - for [0.1s, 0.2s, 0.4s, ...] between retries. It will never be longer + for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer than :attr:`Retry.BACKOFF_MAX`. By default, backoff is disabled (set to 0). diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py index e8d9e7d2..4a64d7ef 100644 --- a/requests/packages/urllib3/util/ssl_.py +++ b/requests/packages/urllib3/util/ssl_.py @@ -117,7 +117,7 @@ except ImportError: 'urllib3 from configuring SSL appropriately and may cause ' 'certain SSL connections to fail. You can upgrade to a newer ' 'version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.org/en/latest/security.html' + 'https://urllib3.readthedocs.io/en/latest/security.html' '#insecureplatformwarning.', InsecurePlatformWarning ) @@ -313,7 +313,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, 'This may cause the server to present an incorrect TLS ' 'certificate, which can cause validation failures. You can upgrade to ' 'a newer version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.org/en/latest/security.html' + 'https://urllib3.readthedocs.io/en/latest/security.html' '#snimissingwarning.', SNIMissingWarning ) From c6eea58081dca411a3bd260d208ab3fbaaf512f1 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 8 Aug 2016 13:37:09 +0100 Subject: [PATCH 28/56] Changelog for 2.11 --- HISTORY.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 8913c8c0..2352815a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,30 @@ Release History --------------- +2.11.0 (2016-08-XX) ++++++++++++++++++++ + +**Improvements** + +- Added support for the ``ALL_PROXY`` environment variable. +- Reject header values that contain leading whitespace or newline characters to + reduce risk of header smuggling. + +**Bugfixes** + +- Fixed occasional ``TypeError`` when attempting to decode a JSON response that + occurred in an error case. Now correctly returns a ``ValueError``. +- Requests would incorrectly ignore a non-CIDR IP address in the ``NO_PROXY`` + environment variables: Requests now treats it as a specific IP. +- Fixed a bug when sending JSON data that could cause us to encounter obscure + OpenSSL errors in certain network conditions (yes, really). +- Added type checks to ensure that ``iter_content`` only accepts integers and + ``None`` for chunk sizes. +- Fixed issue where responses whose body had not been fully consumed would have + the underlying connection closed but not returned to the connection pool, + which could cause Requests to hang in situations where the ``HTTPAdapter`` + had been configured to use a blocking connection pool. + 2.10.0 (2016-04-29) +++++++++++++++++++ From d7700bebeb2094ef47e6e2a6e20a536244ad9871 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 8 Aug 2016 13:37:37 +0100 Subject: [PATCH 29/56] Oh yeah, we updated urllib3 too. --- HISTORY.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 2352815a..69a6318d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,10 @@ Release History which could cause Requests to hang in situations where the ``HTTPAdapter`` had been configured to use a blocking connection pool. +**Miscellaneous** + +- Updated bundled urllib3 to 1.16. + 2.10.0 (2016-04-29) +++++++++++++++++++ From a2e41ba1f1ae81b2ed224030df908a1432ad435e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 8 Aug 2016 11:06:58 -0400 Subject: [PATCH 30/56] v2.11.0 --- HISTORY.rst | 2 +- requests/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 69a6318d..21a093ed 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ Release History --------------- -2.11.0 (2016-08-XX) +2.11.0 (2016-08-08) +++++++++++++++++++ **Improvements** diff --git a/requests/__init__.py b/requests/__init__.py index aea1fbf2..abcdaaa8 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -41,8 +41,8 @@ is at . """ __title__ = 'requests' -__version__ = '2.10.0' -__build__ = 0x021000 +__version__ = '2.11.0' +__build__ = 0x021100 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016 Kenneth Reitz' From 57e7a308b0b118b8fb98b4f453b5653d245d5a52 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 8 Aug 2016 17:35:40 -0400 Subject: [PATCH 31/56] Update HISTORY.rst --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 21a093ed..6e0e01ed 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,6 +30,7 @@ Release History **Miscellaneous** - Updated bundled urllib3 to 1.16. +- Some previous releases accidentally accepted integers as acceptable header values. This release does not. 2.10.0 (2016-04-29) +++++++++++++++++++ From bab4aa0fbf765a85b93f07d01cbc45d064d3bf72 Mon Sep 17 00:00:00 2001 From: David Poggi Date: Mon, 8 Aug 2016 22:49:35 -0400 Subject: [PATCH 32/56] Clarify Python versions supported by requests --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 5eb643e1..81eb8f8c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -87,7 +87,7 @@ Requests is ready for today's web. - Chunked Requests - Thread-safety -Requests supports Python 2.6 — 3.5, and runs great on PyPy. +Requests supports Python 2.6-2.7 and 3.3-3.5, and runs great on PyPy. The User Guide From 33a74f1787fd9b55defa3f7afdd2cd1463dae7b4 Mon Sep 17 00:00:00 2001 From: David Poggi Date: Mon, 8 Aug 2016 22:57:05 -0400 Subject: [PATCH 33/56] Update index.rst Range formatting update. --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 81eb8f8c..2706b01f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -87,7 +87,7 @@ Requests is ready for today's web. - Chunked Requests - Thread-safety -Requests supports Python 2.6-2.7 and 3.3-3.5, and runs great on PyPy. +Requests supports Python 2.6 — 2.7 and 3.3 — 3.5, and runs great on PyPy. The User Guide From 5f7b8ab0a7af12d4491542677a6ad5a32c24d1d1 Mon Sep 17 00:00:00 2001 From: David Poggi Date: Mon, 8 Aug 2016 22:58:26 -0400 Subject: [PATCH 34/56] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 2706b01f..d1cbc856 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -87,7 +87,7 @@ Requests is ready for today's web. - Chunked Requests - Thread-safety -Requests supports Python 2.6 — 2.7 and 3.3 — 3.5, and runs great on PyPy. +Requests officially supports Python 2.6 — 2.7 and 3.3 — 3.5, and runs great on PyPy. The User Guide From 931397829e788fd94125a072deaf4882a6afbc4d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 8 Aug 2016 23:01:48 -0400 Subject: [PATCH 35/56] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index d1cbc856..a1aa9d54 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -87,7 +87,7 @@ Requests is ready for today's web. - Chunked Requests - Thread-safety -Requests officially supports Python 2.6 — 2.7 and 3.3 — 3.5, and runs great on PyPy. +Requests officially supports Python 2.6–2.7 & 3.3–3.5, and runs great on PyPy. The User Guide From 542fbbc67fd07819551012295ff5468eb2714f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Tue, 9 Aug 2016 13:32:56 +0300 Subject: [PATCH 36/56] Document bunch of return types --- requests/adapters.py | 6 ++++ requests/auth.py | 9 +++++- requests/cookies.py | 10 ++++++- requests/sessions.py | 34 ++++++++++++++++++++--- requests/utils.py | 65 ++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 113 insertions(+), 11 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 75c7901e..4a4c4e0e 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -168,6 +168,7 @@ class HTTPAdapter(BaseAdapter): :param proxy: The proxy to return a urllib3 ProxyManager for. :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager. :returns: ProxyManager + :rtype: requests.packages.urllib3.ProxyManager """ if proxy in self.proxy_manager: manager = self.proxy_manager[proxy] @@ -244,6 +245,7 @@ class HTTPAdapter(BaseAdapter): :param req: The :class:`PreparedRequest ` used to generate the response. :param resp: The urllib3 response object. + :rtype: requests.Response """ response = Response() @@ -279,6 +281,7 @@ class HTTPAdapter(BaseAdapter): :param url: The URL to connect to. :param proxies: (optional) A Requests-style dictionary of proxies used on this request. + :rtype: requests.packages.urllib3.ConnectionPool """ proxy = select_proxy(url, proxies) @@ -316,6 +319,7 @@ class HTTPAdapter(BaseAdapter): :param request: The :class:`PreparedRequest ` being sent. :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs. + :rtype: str """ proxy = select_proxy(request.url, proxies) scheme = urlparse(request.url).scheme @@ -357,6 +361,7 @@ class HTTPAdapter(BaseAdapter): :class:`HTTPAdapter `. :param proxies: The url of the proxy being used for this request. + :rtype: dict """ headers = {} username, password = get_auth_from_url(proxy) @@ -379,6 +384,7 @@ class HTTPAdapter(BaseAdapter): :param verify: (optional) Whether to verify SSL certificates. :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. + :rtype: requests.Response """ conn = self.get_connection(request.url, proxies) diff --git a/requests/auth.py b/requests/auth.py index 4f09b911..49bcb24a 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -90,6 +90,9 @@ class HTTPDigestAuth(AuthBase): self._thread_local.num_401_calls = None def build_digest_header(self, method, url): + """ + :rtype: str + """ realm = self._thread_local.chal['realm'] nonce = self._thread_local.chal['nonce'] @@ -182,7 +185,11 @@ class HTTPDigestAuth(AuthBase): self._thread_local.num_401_calls = 1 def handle_401(self, r, **kwargs): - """Takes the given response and tries digest-auth, if needed.""" + """ + Takes the given response and tries digest-auth, if needed. + + :rtype: requests.Response + """ if self._thread_local.pos is not None: # Rewind the file position indicator of the body to where diff --git a/requests/cookies.py b/requests/cookies.py index a133684a..41a2fde1 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -134,7 +134,11 @@ def extract_cookies_to_jar(jar, request, response): def get_cookie_header(jar, request): - """Produce an appropriate Cookie header string to be sent with `request`, or None.""" + """ + Produce an appropriate Cookie header string to be sent with `request`, or None. + + :rtype: str + """ r = MockRequest(request) jar.add_cookie_header(r) return r.get_new_headers().get('Cookie') @@ -283,6 +287,8 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): def multiple_domains(self): """Returns True if there are multiple domains in the jar. Returns False otherwise. + + :rtype: bool """ domains = [] for cookie in iter(self): @@ -295,6 +301,8 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): """Takes as an argument an optional domain and path and returns a plain old Python dict of name-value pairs of cookies that meet the requirements. + + :rtype: dict """ dictionary = {} for cookie in iter(self): diff --git a/requests/sessions.py b/requests/sessions.py index d8b11fad..8d8d9105 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -214,6 +214,8 @@ class SessionRedirectMixin(object): This method also replaces the Proxy-Authorization header where necessary. + + :rtype: dict """ headers = prepared_request.headers url = prepared_request.url @@ -360,6 +362,7 @@ class Session(SessionRedirectMixin): :param request: :class:`Request` instance to prepare with this session's settings. + :rtype: requests.PreparedRequest """ cookies = request.cookies or {} @@ -477,6 +480,7 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response """ kwargs.setdefault('allow_redirects', True) @@ -487,6 +491,7 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response """ kwargs.setdefault('allow_redirects', True) @@ -497,6 +502,7 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response """ kwargs.setdefault('allow_redirects', False) @@ -509,6 +515,7 @@ class Session(SessionRedirectMixin): :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response """ return self.request('POST', url, data=data, json=json, **kwargs) @@ -519,6 +526,7 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response """ return self.request('PUT', url, data=data, **kwargs) @@ -529,6 +537,7 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response """ return self.request('PATCH', url, data=data, **kwargs) @@ -538,12 +547,17 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response """ return self.request('DELETE', url, **kwargs) def send(self, request, **kwargs): - """Send a given PreparedRequest.""" + """ + Send a given PreparedRequest. + + :rtype: requests.Response + """ # Set defaults that the hooks can utilize to ensure they always have # the correct parameters to reproduce the previous request. kwargs.setdefault('stream', self.stream) @@ -615,7 +629,11 @@ class Session(SessionRedirectMixin): return r def merge_environment_settings(self, url, proxies, stream, verify, cert): - """Check the environment and merge it with some settings.""" + """ + Check the environment and merge it with some settings. + + :rtype: dict + """ # Gather clues from the surrounding environment. if self.trust_env: # Set environment's proxies. @@ -639,7 +657,11 @@ class Session(SessionRedirectMixin): 'cert': cert} def get_adapter(self, url): - """Returns the appropriate connection adapter for the given URL.""" + """ + Returns the appropriate connection adapter for the given URL. + + :rtype: requests.adapters.BaseAdapter + """ for (prefix, adapter) in self.adapters.items(): if url.lower().startswith(prefix): @@ -680,6 +702,10 @@ class Session(SessionRedirectMixin): def session(): - """Returns a :class:`Session` for context-management.""" + """ + Returns a :class:`Session` for context-management. + + :rtype: Session + """ return Session() diff --git a/requests/utils.py b/requests/utils.py index 1c2a1648..e37b9109 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -164,6 +164,8 @@ def from_key_val_list(value): ValueError: need more than 1 value to unpack >>> from_key_val_list({'key': 'val'}) OrderedDict([('key', 'val')]) + + :rtype: OrderedDict """ if value is None: return None @@ -186,6 +188,8 @@ def to_key_val_list(value): [('key', 'val')] >>> to_key_val_list('string') ValueError: cannot encode objects that are not 2-tuples. + + :rtype: list """ if value is None: return None @@ -221,6 +225,7 @@ def parse_list_header(value): :param value: a string with a list header. :return: :class:`list` + :rtype: list """ result = [] for item in _parse_list_header(value): @@ -251,6 +256,7 @@ def parse_dict_header(value): :param value: a string with a dict header. :return: :class:`dict` + :rtype: dict """ result = {} for item in _parse_list_header(value): @@ -271,6 +277,7 @@ def unquote_header_value(value, is_filename=False): using for quoting. :param value: the header value to unquote. + :rtype: str """ if value and value[0] == value[-1] == '"': # this is not the real unquoting, but fixing this so that the @@ -293,6 +300,7 @@ def dict_from_cookiejar(cj): """Returns a key/value dictionary from a CookieJar. :param cj: CookieJar object to extract cookies from. + :rtype: dict """ cookie_dict = {} @@ -308,6 +316,7 @@ def add_dict_to_cookiejar(cj, cookie_dict): :param cj: CookieJar to insert cookies into. :param cookie_dict: Dict of key/values to insert into CookieJar. + :rtype: CookieJar """ cj2 = cookiejar_from_dict(cookie_dict) @@ -339,6 +348,7 @@ def get_encoding_from_headers(headers): """Returns encodings from given HTTP Header Dict. :param headers: dictionary to extract encoding from. + :rtype: str """ content_type = headers.get('content-type') @@ -399,6 +409,8 @@ def get_unicode_from_response(r): 1. charset from content-type 2. fall back and replace all unicode characters + + :rtype: str """ warnings.warn(( 'In requests 3.0, get_unicode_from_response will be removed. For ' @@ -433,6 +445,8 @@ UNRESERVED_SET = frozenset( def unquote_unreserved(uri): """Un-escape any percent-escape sequences in a URI that are unreserved characters. This leaves all reserved, illegal and non-ASCII bytes encoded. + + :rtype: str """ parts = uri.split('%') for i in range(1, len(parts)): @@ -457,6 +471,8 @@ def requote_uri(uri): This function passes the given URI through an unquote/quote cycle to ensure that it is fully and consistently quoted. + + :rtype: str """ safe_with_percent = "!#$%&'()*+,/:;=?@[]~" safe_without_percent = "!#$&'()*+,/:;=?@[]~" @@ -477,6 +493,8 @@ def address_in_network(ip, net): Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24 returns False if ip = 192.168.1.1 and net = 192.168.100.0/24 + + :rtype: bool """ ipaddr = struct.unpack('=L', socket.inet_aton(ip))[0] netaddr, bits = net.split('/') @@ -489,12 +507,17 @@ def dotted_netmask(mask): """Converts mask from /xx format to xxx.xxx.xxx.xxx Example: if mask is 24 function returns 255.255.255.0 + + :rtype: str """ bits = 0xffffffff ^ (1 << 32 - mask) - 1 return socket.inet_ntoa(struct.pack('>I', bits)) def is_ipv4_address(string_ip): + """ + :rtype: bool + """ try: socket.inet_aton(string_ip) except socket.error: @@ -503,7 +526,11 @@ def is_ipv4_address(string_ip): def is_valid_cidr(string_network): - """Very simple check of the cidr format in no_proxy variable""" + """ + Very simple check of the cidr format in no_proxy variable. + + :rtype: bool + """ if string_network.count('/') == 1: try: mask = int(string_network.split('/')[1]) @@ -523,7 +550,11 @@ def is_valid_cidr(string_network): def should_bypass_proxies(url): - """Returns whether we should bypass proxies or not.""" + """ + Returns whether we should bypass proxies or not. + + :rtype: bool + """ get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) # First check whether no_proxy is defined. If it is, check that the URL @@ -573,7 +604,11 @@ def should_bypass_proxies(url): def get_environ_proxies(url): - """Return a dict of environment proxies.""" + """ + Return a dict of environment proxies. + + :rtype: dict + """ if should_bypass_proxies(url): return {} else: @@ -607,11 +642,18 @@ def select_proxy(url, proxies): def default_user_agent(name="python-requests"): - """Return a string representing the default user agent.""" + """ + Return a string representing the default user agent. + + :rtype: str + """ return '%s/%s' % (name, __version__) def default_headers(): + """ + :rtype: requests.structures.CaseInsensitiveDict + """ return CaseInsensitiveDict({ 'User-Agent': default_user_agent(), 'Accept-Encoding': ', '.join(('gzip', 'deflate')), @@ -624,6 +666,8 @@ def parse_header_links(value): """Return a dict of parsed link headers proxies. i.e. Link: ; rel=front; type="image/jpeg",; rel=back;type="image/jpeg" + + :rtype: list """ links = [] @@ -658,6 +702,9 @@ _null3 = _null * 3 def guess_json_utf(data): + """ + :rtype: str + """ # JSON always starts with two ASCII characters, so detection is as # easy as counting the nulls and from their location and count # determine the encoding. Also detect a BOM, if present. @@ -689,6 +736,8 @@ def guess_json_utf(data): def prepend_scheme_if_needed(url, new_scheme): """Given a URL that may or may not have a scheme, prepend the given scheme. Does not replace a present scheme with the one provided as an argument. + + :rtype: str """ scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) @@ -704,6 +753,8 @@ def prepend_scheme_if_needed(url, new_scheme): def get_auth_from_url(url): """Given a url with authentication components, extract them into a tuple of username,password. + + :rtype: (str,str) """ parsed = urlparse(url) @@ -757,7 +808,11 @@ def check_header_validity(header): def urldefragauth(url): - """Given a url remove the fragment and the authentication part""" + """ + Given a url remove the fragment and the authentication part. + + :rtype: str + """ scheme, netloc, path, params, query, fragment = urlparse(url) # see func:`prepend_scheme_if_needed` From d7f56ba9383575a6b7d361db0123a93c70a2b42f Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Tue, 9 Aug 2016 07:36:39 -0600 Subject: [PATCH 37/56] reverting 3362 --- requests/utils.py | 17 +++++------------ tests/test_requests.py | 7 ------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index e37b9109..dfeb77d9 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -367,20 +367,13 @@ def get_encoding_from_headers(headers): def stream_decode_response_unicode(iterator, r): """Stream decodes a iterator.""" - encoding = r.encoding - if encoding is None: - encoding = r.apparent_encoding - - try: - decoder = codecs.getincrementaldecoder(encoding)(errors='replace') - except (LookupError, TypeError): - # A LookupError is raised if the encoding was not found which could - # indicate a misspelling or similar mistake. - # - # A TypeError can be raised if encoding is None - raise UnicodeError("Unable to decode contents with encoding %s." % encoding) + if r.encoding is None: + for item in iterator: + yield item + return + decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace') for chunk in iterator: rv = decoder.decode(chunk) if rv: diff --git a/tests/test_requests.py b/tests/test_requests.py index 3d959a4b..efbc7ed0 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -973,13 +973,6 @@ class TestRequests: chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) - # check for encoding value of None - r = requests.Response() - r.raw = io.BytesIO(b'the content') - r.encoding = None - chunks = r.iter_content(decode_unicode=True) - assert all(isinstance(chunk, str) for chunk in chunks) - def test_response_reason_unicode(self): # check for unicode HTTP status r = requests.Response() From 1e253cd0b35c97605fb6aab636e4ce0be32f9686 Mon Sep 17 00:00:00 2001 From: David Poggi Date: Tue, 9 Aug 2016 11:14:40 -0400 Subject: [PATCH 38/56] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d072d1e8..fbcb393f 100644 --- a/README.rst +++ b/README.rst @@ -61,7 +61,7 @@ Requests is ready for today's web. - Chunked Requests - Thread-safety -Requests supports Python 2.6 — 3.5, and runs great on PyPy. +Requests officially supports Python 2.6–2.7 & 3.3–3.5, and runs great on PyPy. Installation ------------ From 10bc2f05d322fb3ec8559cf9ee1fc88c071f74a1 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Tue, 9 Aug 2016 15:12:34 -0600 Subject: [PATCH 39/56] updating HISTORY --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6e0e01ed..faa9364a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,7 +30,7 @@ Release History **Miscellaneous** - Updated bundled urllib3 to 1.16. -- Some previous releases accidentally accepted integers as acceptable header values. This release does not. +- Some previous releases accidentally accepted non-strings as acceptable header values. This release does not. 2.10.0 (2016-04-29) +++++++++++++++++++ From e50c61bc866ff4358e6ef320b4f1f31b979bd45e Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Wed, 10 Aug 2016 11:02:26 -0600 Subject: [PATCH 40/56] removing Content-Type and Transfer-Encoding headers on redirect --- requests/sessions.py | 7 ++++--- tests/test_requests.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 8d8d9105..bcbcc880 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -140,9 +140,10 @@ class SessionRedirectMixin(object): # https://github.com/kennethreitz/requests/issues/1084 if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect): - if 'Content-Length' in prepared_request.headers: - del prepared_request.headers['Content-Length'] - + # https://github.com/kennethreitz/requests/issues/3490 + purged_headers = ('Content-Length', 'Content-Type', 'Transfer-Encoding') + for header in purged_headers: + prepared_request.headers.pop(header, None) prepared_request.body = None headers = prepared_request.headers diff --git a/tests/test_requests.py b/tests/test_requests.py index efbc7ed0..3f791334 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -221,6 +221,41 @@ class TestRequests: assert r.history[0].status_code == 303 assert r.history[0].is_redirect + def test_header_and_body_removal_on_redirect(self, httpbin): + purged_headers = ('Content-Length', 'Content-Type') + ses = requests.Session() + req = requests.Request('POST', httpbin('post'), data={'test': 'data'}) + prep = ses.prepare_request(req) + resp = ses.send(prep) + + # Mimic a redirect response + resp.status_code = 302 + resp.headers['location'] = 'get' + + # Run request through resolve_redirects + next_resp = next(ses.resolve_redirects(resp, prep)) + assert next_resp.request.body is None + for header in purged_headers: + assert header not in next_resp.request.headers + + def test_transfer_enc_removal_on_redirect(self, httpbin): + purged_headers = ('Transfer-Encoding', 'Content-Type') + ses = requests.Session() + req = requests.Request('POST', httpbin('post'), data=(b'x' for x in range(1))) + prep = ses.prepare_request(req) + assert 'Transfer-Encoding' in prep.headers + resp = ses.send(prep) + + # Mimic a redirect response + resp.status_code = 302 + resp.headers['location'] = 'get' + + # Run request through resolve_redirect + next_resp = next(ses.resolve_redirects(resp, prep)) + assert next_resp.request.body is None + for header in purged_headers: + assert header not in next_resp.request.headers + def test_HTTP_200_OK_GET_WITH_PARAMS(self, httpbin): heads = {'User-agent': 'Mozilla/5.0'} From 4d5091c2875d487c1d5ca958ed8e6f3397e042fe Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Thu, 11 Aug 2016 14:51:38 -0600 Subject: [PATCH 41/56] building Response manually --- tests/test_requests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_requests.py b/tests/test_requests.py index 3f791334..dcf14eec 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -244,11 +244,16 @@ class TestRequests: req = requests.Request('POST', httpbin('post'), data=(b'x' for x in range(1))) prep = ses.prepare_request(req) assert 'Transfer-Encoding' in prep.headers - resp = ses.send(prep) + + # Create Response to avoid https://github.com/kevin1024/pytest-httpbin/issues/33 + resp = requests.Response() + resp.raw = io.BytesIO(b'the content') + resp.request = prep + setattr(resp.raw, 'release_conn', lambda *args: args) # Mimic a redirect response resp.status_code = 302 - resp.headers['location'] = 'get' + resp.headers['location'] = httpbin('get') # Run request through resolve_redirect next_resp = next(ses.resolve_redirects(resp, prep)) From 59f12c9669c53c5cb110870cc94e8181831f45ab Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Thu, 11 Aug 2016 16:32:19 -0600 Subject: [PATCH 42/56] adding passthrough in close() for non-urllib3-like Responses --- requests/models.py | 4 +++- tests/test_requests.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 11434ef4..74385674 100644 --- a/requests/models.py +++ b/requests/models.py @@ -870,4 +870,6 @@ class Response(object): if not self._content_consumed: self.raw.close() - return self.raw.release_conn() + release_conn = getattr(self.raw, 'release_conn', None) + if release_conn is not None: + release_conn() diff --git a/tests/test_requests.py b/tests/test_requests.py index efbc7ed0..83338fc3 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1353,6 +1353,15 @@ class TestRequests: with pytest.raises(ValueError): r.json() + def test_response_without_release_conn(self): + """Test `close` call for non-urllib3-like raw objects. + Should work when `release_conn` attr doesn't exist on `response.raw`. + """ + resp = requests.Response() + resp.raw = StringIO.StringIO('test') + assert not resp.raw.closed + resp.close() + assert resp.raw.closed class TestCaseInsensitiveDict: From 00f83fb0c64aa48a27a3da2df590530ef7eeff85 Mon Sep 17 00:00:00 2001 From: Jiayuan Zhang Date: Sat, 13 Aug 2016 10:19:10 +0800 Subject: [PATCH 43/56] fix grammer mistakes in Quickstart --- docs/user/quickstart.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index ea87eecc..b31276e6 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -132,9 +132,9 @@ For example, to create an image from binary data returned by a request, you can use the following code:: >>> from PIL import Image - >>> from io import StringIO + >>> from io import BytesIO - >>> i = Image.open(StringIO(r.content)) + >>> i = Image.open(BytesIO(r.content)) JSON Response Content @@ -416,7 +416,7 @@ parameter:: >>> r = requests.get(url, cookies=cookies) >>> r.text '{"cookies": {"cookies_are": "working"}}' - + Cookies are returned in a :class:`~requests.cookies.RequestsCookieJar`, which acts like a ``dict`` but also offers a more complete interface, suitable for use over multiple domains or paths. Cookie jars can From 1435cf5affcc3822f8cb4e424bef0260083bfce5 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Sun, 14 Aug 2016 20:31:21 -0600 Subject: [PATCH 44/56] docs updates --- docs/api.rst | 3 ++ docs/community/faq.rst | 6 ++-- docs/community/out-there.rst | 11 ------- docs/dev/todo.rst | 2 +- docs/user/advanced.rst | 60 +++++++++++++++++------------------- docs/user/quickstart.rst | 20 ++++++------ 6 files changed, 45 insertions(+), 57 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 08e2b6ee..31f763c4 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -63,6 +63,9 @@ Lower-Lower-Level Classes .. autoclass:: requests.PreparedRequest :inherited-members: +.. autoclass:: requests.adapters.BaseAdapter + :inherited-members: + .. autoclass:: requests.adapters.HTTPAdapter :inherited-members: diff --git a/docs/community/faq.rst b/docs/community/faq.rst index f869ee9a..c87687af 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -56,12 +56,10 @@ supported: * Python 2.6 * Python 2.7 -* Python 3.1 -* Python 3.2 * Python 3.3 * Python 3.4 -* PyPy 1.9 -* PyPy 2.2 +* Python 3.5 +* PyPy What are "hostname doesn't match" errors? ----------------------------------------- diff --git a/docs/community/out-there.rst b/docs/community/out-there.rst index de41f1d4..645c0ac4 100644 --- a/docs/community/out-there.rst +++ b/docs/community/out-there.rst @@ -1,17 +1,6 @@ Integrations ============ -ScraperWiki ------------- - -`ScraperWiki `_ is an excellent service that allows -you to run Python, Ruby, and PHP scraper scripts on the web. Now, Requests -v0.6.1 is available to use in your scrapers! - -To give it a try, simply:: - - import requests - Python for iOS -------------- diff --git a/docs/dev/todo.rst b/docs/dev/todo.rst index 5f1700a9..79b95a21 100644 --- a/docs/dev/todo.rst +++ b/docs/dev/todo.rst @@ -41,7 +41,7 @@ Requests currently supports the following versions of Python: - Python 3.3 - Python 3.4 - Python 3.5 -- PyPy 1.9 +- PyPy Google AppEngine is not officially supported although support is available with the `Requests-Toolbelt`_. diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index b53fa436..d6d04569 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -87,11 +87,11 @@ See the :ref:`Session API Docs ` to learn more. Request and Response Objects ---------------------------- -Whenever a call is made to ``requests.get()`` and friends you are doing two +Whenever a call is made to ``requests.get()`` and friends, you are doing two major things. First, you are constructing a ``Request`` object which will be sent off to a server to request or query some resource. Second, a ``Response`` -object is generated once ``requests`` gets a response back from the server. -The Response object contains all of the information returned by the server and +object is generated once Requests gets a response back from the server. +The ``Response`` object contains all of the information returned by the server and also contains the ``Request`` object you created originally. Here is a simple request to get some very important information from Wikipedia's servers:: @@ -213,9 +213,7 @@ You can pass ``verify`` the path to a CA_BUNDLE file or directory with certifica This list of trusted CAs can also be specified through the ``REQUESTS_CA_BUNDLE`` environment variable. -Requests can also ignore verifying the SSL certificate if you set ``verify`` to False. - -:: +Requests can also ignore verifying the SSL certificate if you set ``verify`` to False:: >>> requests.get('https://kennethreitz.com', verify=False) @@ -242,7 +240,7 @@ If you specify a wrong path or an invalid cert, you'll get a SSLError:: CA Certificates --------------- -By default Requests bundles a set of root CAs that it trusts, sourced from the +By default, Requests bundles a set of root CAs that it trusts, sourced from the `Mozilla trust store`_. However, these are only updated once for each Requests version. This means that if you pin a Requests version your certificates can become extremely out of date. @@ -266,7 +264,7 @@ Body Content Workflow By default, when you make a request, the body of the response is downloaded immediately. You can override this behaviour and defer downloading the response -body until you access the :class:`Response.content ` +body until you access the :attr:`Response.content ` attribute with the ``stream`` parameter:: tarball_url = 'https://github.com/kennethreitz/requests/tarball/master' @@ -279,15 +277,15 @@ remains open, hence allowing us to make content retrieval conditional:: content = r.content ... -You can further control the workflow by use of the :class:`Response.iter_content ` -and :class:`Response.iter_lines ` methods. +You can further control the workflow by use of the :meth:`Response.iter_content() ` +and :meth:`Response.iter_lines() ` methods. Alternatively, you can read the undecoded body from the underlying urllib3 :class:`urllib3.HTTPResponse ` at -:class:`Response.raw `. +:attr:`Response.raw `. If you set ``stream`` to ``True`` when making a request, Requests cannot release the connection back to the pool unless you consume all the data or call -:class:`Response.close `. This can lead to +:meth:`Response.close `. This can lead to inefficiency with connections. If you find yourself partially reading request bodies (or not reading them at all) while using ``stream=True``, you should consider using ``contextlib.closing`` (`documented here`_), like this:: @@ -349,11 +347,11 @@ a length) for your body:: requests.post('http://some.url/chunked', data=gen()) For chunked encoded responses, it's best to iterate over the data using -:meth:`Response.iter_content() `. In +:meth:`Response.iter_content() `. In an ideal situation you'll have set ``stream=True`` on the request, in which -case you can iterate chunk-by-chunk by calling ``iter_content`` with a chunk -size parameter of ``None``. If you want to set a maximum size of the chunk, -you can set a chunk size parameter to any integer. +case you can iterate chunk-by-chunk by calling ``iter_content`` with a ``chunk_size`` +parameter of ``None``. If you want to set a maximum size of the chunk, +you can set a ``chunk_size`` parameter to any integer. .. _multipart: @@ -440,9 +438,10 @@ Requests allows you to use specify your own authentication mechanism. Any callable which is passed as the ``auth`` argument to a request method will have the opportunity to modify the request before it is dispatched. -Authentication implementations are subclasses of ``requests.auth.AuthBase``, +Authentication implementations are subclasses of :class:`AuthBase `, and are easy to define. Requests provides two common authentication scheme -implementations in ``requests.auth``: ``HTTPBasicAuth`` and ``HTTPDigestAuth``. +implementations in ``requests.auth``: :class:`HTTPBasicAuth ` and +:class:`HTTPDigestAuth `. Let's pretend that we have a web service that will only respond if the ``X-Pizza`` header is set to a password value. Unlikely, but just go with it. @@ -472,11 +471,11 @@ Then, we can make a request using our Pizza Auth:: Streaming Requests ------------------ -With :class:`requests.Response.iter_lines()` you can easily +With :meth:`Response.iter_lines() ` you can easily iterate over streaming APIs such as the `Twitter Streaming API `_. Simply set ``stream`` to ``True`` and iterate over the response with -:class:`~requests.Response.iter_lines()`:: +:meth:`~requests.Response.iter_lines()`:: import json import requests @@ -491,7 +490,7 @@ set ``stream`` to ``True`` and iterate over the response with .. warning:: - :class:`~requests.Response.iter_lines()` is not reentrant safe. + :meth:`~requests.Response.iter_lines()` is not reentrant safe. Calling this method multiple times causes some of the received data being lost. In case you need to call it from multiple places, use the resulting iterator object instead:: @@ -669,8 +668,9 @@ commits is POST, which creates a new commit. As we're using the Requests repo, we should probably avoid making ham-handed POSTS to it. Instead, let's play with the Issues feature of GitHub. -This documentation was added in response to Issue #482. Given that this issue -already exists, we will use it as an example. Let's start by getting it. +This documentation was added in response to +`Issue #482 `_. Given that +this issue already exists, we will use it as an example. Let's start by getting it. :: @@ -845,8 +845,8 @@ with the given prefix will use the given Transport Adapter. Many of the details of implementing a Transport Adapter are beyond the scope of this documentation, but take a look at the next example for a simple SSL use- -case. For more than that, you might look at subclassing -``requests.adapters.BaseAdapter``. +case. For more than that, you might look at subclassing the +:class:`BaseAdapter `. Example: Specific SSL Version ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -858,10 +858,8 @@ that uses a version that isn't compatible with the default. You can use Transport Adapters for this by taking most of the existing implementation of HTTPAdapter, and adding a parameter *ssl_version* that gets -passed-through to `urllib3`. We'll make a TA that instructs the library to use -SSLv3: - -:: +passed-through to `urllib3`. We'll make a Transport Adapter that instructs the +library to use SSLv3:: import ssl @@ -904,7 +902,7 @@ Header Ordering In unusual circumstances you may want to provide headers in an ordered manner. If you pass an ``OrderedDict`` to the ``headers`` keyword argument, that will provide the headers with an ordering. *However*, the ordering of the default headers used by Requests will be preferred, which means that if you override default headers in the ``headers`` keyword argument, they may appear out of order compared to other headers in that keyword argument. -If this is problematic, users should consider setting the default headers on a :class:`Session ` object, by setting :data:`Session ` to a custom ``OrderedDict``. That ordering will always be preferred. +If this is problematic, users should consider setting the default headers on a :class:`Session ` object, by setting :attr:`Session ` to a custom ``OrderedDict``. That ordering will always be preferred. .. _timeouts: @@ -940,7 +938,7 @@ If the remote server is very slow, you can tell Requests to wait forever for a response, by passing None as a timeout value and then retrieving a cup of coffee. -.. code-block:: python +:: r = requests.get('https://github.com', timeout=None) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index b31276e6..0d725389 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -27,7 +27,7 @@ Begin by importing the Requests module:: >>> import requests Now, let's try to get a webpage. For this example, let's get GitHub's public -timeline :: +timeline:: >>> r = requests.get('https://api.github.com/events') @@ -148,11 +148,11 @@ There's also a builtin JSON decoder, in case you're dealing with JSON data:: >>> r.json() [{u'repository': {u'open_issues': 0, u'url': 'https://github.com/... -In case the JSON decoding fails, ``r.json`` raises an exception. For example, if +In case the JSON decoding fails, ``r.json()`` raises an exception. For example, if the response gets a 204 (No Content), or if the response contains invalid JSON, -attempting ``r.json`` raises ``ValueError: No JSON object could be decoded``. +attempting ``r.json()`` raises ``ValueError: No JSON object could be decoded``. -It should be noted that the success of the call to ``r.json`` does **not** +It should be noted that the success of the call to ``r.json()`` does **not** indicate the success of the response. Some servers may return a JSON object in a failed response (e.g. error details with HTTP 500). Such JSON will be decoded and returned. To check that a request is successful, use @@ -439,7 +439,7 @@ HEAD. We can use the ``history`` property of the Response object to track redirection. -The :meth:`Response.history ` list contains the +The :attr:`Response.history ` list contains the :class:`Response ` objects that were created in order to complete the request. The list is sorted from the oldest to the most recent response. @@ -504,20 +504,20 @@ Errors and Exceptions --------------------- In the event of a network problem (e.g. DNS failure, refused connection, etc), -Requests will raise a :class:`~requests.exceptions.ConnectionError` exception. +Requests will raise a :exc:`~requests.exceptions.ConnectionError` exception. :meth:`Response.raise_for_status() ` will -raise an :class:`~requests.exceptions.HTTPError` if the HTTP request +raise an :exc:`~requests.exceptions.HTTPError` if the HTTP request returned an unsuccessful status code. -If a request times out, a :class:`~requests.exceptions.Timeout` exception is +If a request times out, a :exc:`~requests.exceptions.Timeout` exception is raised. If a request exceeds the configured number of maximum redirections, a -:class:`~requests.exceptions.TooManyRedirects` exception is raised. +:exc:`~requests.exceptions.TooManyRedirects` exception is raised. All exceptions that Requests explicitly raises inherit from -:class:`requests.exceptions.RequestException`. +:exc:`requests.exceptions.RequestException`. ----------------------- From ab1a6a4d0057e8eb1ccd70c580d78a8bac3506ea Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 15 Aug 2016 10:05:37 +0100 Subject: [PATCH 45/56] Prepare changelog for 2.11.1. --- HISTORY.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index faa9364a..b694147a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,17 @@ Release History --------------- +2.11.1 (2016-08-XX) ++++++++++++++++++++ + +**Bugfixes** + +- Fixed a bug when using ``iter_content`` with ``decode_unicode=True`` for + streamed bodies would raise ``AttributeError``. This bug was introduced in + 2.11. +- Strip Content-Type and Transfer-Encoding headers from the header block when + following a redirect that transforms the verb from POST/PUT to GET. + 2.11.0 (2016-08-08) +++++++++++++++++++ From b9648bad19060b06ae3e350add499548580c58ad Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 15 Aug 2016 15:36:41 -0400 Subject: [PATCH 46/56] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index a1aa9d54..d8279a9c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -41,7 +41,7 @@ which is embedded within Requests. User Testimonials ----------------- -Her Majesty's Government, Amazon, Google, Twilio, Runscope, Mozilla, Heroku, +The NSA, Her Majesty's Government, Amazon, Google, Twilio, Runscope, Mozilla, Heroku, PayPal, NPR, Obama for America, Transifex, Native Instruments, The Washington Post, Twitter, SoundCloud, Kippt, Readability, Sony, and Federal U.S. Institutions that prefer to be unnamed claim to use Requests internally. From b26606cc3cb5c598819bc3be474368e7d4ecc0d2 Mon Sep 17 00:00:00 2001 From: Maik Himstedt Date: Wed, 17 Aug 2016 20:27:00 +0200 Subject: [PATCH 47/56] Adding notes about Request's timeout behavior. --- AUTHORS.rst | 3 +++ docs/user/advanced.rst | 3 ++- docs/user/quickstart.rst | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index b0ddcabb..ca062664 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -159,11 +159,14 @@ Patches and Suggestions - Muhammad Yasoob Ullah Khalid (`@yasoob `_) - Paul van der Linden (`@pvanderlinden `_) - Colin Dickson (`@colindickson `_) +- Sabari Kumar Murugesan (`@neosab `_) - Smiley Barry (`@smiley `_) - Shagun Sodhani (`@shagunsodhani `_) - Robin Linderborg (`@vienno `_) - Brian Samek (`@bsamek `_) - Dmitry Dygalo (`@Stranger6667 `_) +- Tomáš Heger (`@geckon `_) - piotrjurkiewicz - Jesse Shapiro (`@haikuginger `_) - Nate Prewitt (`@nateprewitt `_) +- Maik Himstedt diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index d6d04569..3f39f1aa 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -910,7 +910,8 @@ Timeouts -------- Most requests to external servers should have a timeout attached, in case the -server is not responding in a timely manner. Without a timeout, your code may +server is not responding in a timely manner. By standard, requests do not time +out unless a timeout value is set explicitly. Without a timeout, your code may hang for minutes or more. The **connect** timeout is the number of seconds Requests will wait for your diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 0d725389..78134564 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -497,7 +497,8 @@ seconds with the ``timeout`` parameter:: ``timeout`` is not a time limit on the entire response download; rather, an exception is raised if the server has not issued a response for ``timeout`` seconds (more precisely, if no bytes have been - received on the underlying socket for ``timeout`` seconds). + received on the underlying socket for ``timeout`` seconds). If NO timeout is specified explicitly, requests do + not time out. Errors and Exceptions From b5fac316885767b6c00c6b1199de5c8695513fdf Mon Sep 17 00:00:00 2001 From: Maik Date: Wed, 17 Aug 2016 22:03:18 +0200 Subject: [PATCH 48/56] Changes to documentation based on comments and correction of list of authors. --- AUTHORS.rst | 2 -- docs/user/advanced.rst | 2 +- docs/user/quickstart.rst | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index ca062664..0654190b 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -159,13 +159,11 @@ Patches and Suggestions - Muhammad Yasoob Ullah Khalid (`@yasoob `_) - Paul van der Linden (`@pvanderlinden `_) - Colin Dickson (`@colindickson `_) -- Sabari Kumar Murugesan (`@neosab `_) - Smiley Barry (`@smiley `_) - Shagun Sodhani (`@shagunsodhani `_) - Robin Linderborg (`@vienno `_) - Brian Samek (`@bsamek `_) - Dmitry Dygalo (`@Stranger6667 `_) -- Tomáš Heger (`@geckon `_) - piotrjurkiewicz - Jesse Shapiro (`@haikuginger `_) - Nate Prewitt (`@nateprewitt `_) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 3f39f1aa..343ba0ec 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -910,7 +910,7 @@ Timeouts -------- Most requests to external servers should have a timeout attached, in case the -server is not responding in a timely manner. By standard, requests do not time +server is not responding in a timely manner. By default, requests do not time out unless a timeout value is set explicitly. Without a timeout, your code may hang for minutes or more. diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 78134564..c4b739ab 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -497,7 +497,7 @@ seconds with the ``timeout`` parameter:: ``timeout`` is not a time limit on the entire response download; rather, an exception is raised if the server has not issued a response for ``timeout`` seconds (more precisely, if no bytes have been - received on the underlying socket for ``timeout`` seconds). If NO timeout is specified explicitly, requests do + received on the underlying socket for ``timeout`` seconds). If no timeout is specified explicitly, requests do not time out. From 532756803d3b2b7155c31a57f1d5e3cf31224b5d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2016 16:03:38 -0400 Subject: [PATCH 49/56] v2.11.1 --- HISTORY.rst | 2 +- requests/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b694147a..752bd2f3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ Release History --------------- -2.11.1 (2016-08-XX) +2.11.1 (2016-08-17) +++++++++++++++++++ **Bugfixes** diff --git a/requests/__init__.py b/requests/__init__.py index abcdaaa8..9c3b7695 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -41,8 +41,8 @@ is at . """ __title__ = 'requests' -__version__ = '2.11.0' -__build__ = 0x021100 +__version__ = '2.11.1' +__build__ = 0x021101 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016 Kenneth Reitz' From 77b068bb9eb4364ad8c30bedf7654a0d77ebf9ff Mon Sep 17 00:00:00 2001 From: Michael Hunsinger Date: Sun, 21 Aug 2016 10:52:24 -0600 Subject: [PATCH 50/56] Fixed bug to give scheme proxy priority over "all" --- AUTHORS.rst | 1 + requests/utils.py | 4 ++-- tests/test_utils.py | 8 ++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 0654190b..289f1aef 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -168,3 +168,4 @@ Patches and Suggestions - Jesse Shapiro (`@haikuginger `_) - Nate Prewitt (`@nateprewitt `_) - Maik Himstedt +- Michael Hunsinger diff --git a/requests/utils.py b/requests/utils.py index dfeb77d9..e0bb3e7c 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -620,10 +620,10 @@ def select_proxy(url, proxies): return proxies.get('all', proxies.get(urlparts.scheme)) proxy_keys = [ - 'all://' + urlparts.hostname, - 'all', urlparts.scheme + '://' + urlparts.hostname, urlparts.scheme, + 'all://' + urlparts.hostname, + 'all', ] proxy = None for proxy_key in proxy_keys: diff --git a/tests/test_utils.py b/tests/test_utils.py index 2e28c068..b4fc9483 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -323,6 +323,9 @@ http_proxies = {'http': 'http://http.proxy', 'http://some.host': 'http://some.host.proxy'} all_proxies = {'all': 'socks5://http.proxy', 'all://some.host': 'socks5://some.host.proxy'} +mixed_proxies = {'http': 'http://http.proxy', + 'http://some.host': 'http://some.host.proxy', + 'all': 'socks5://http.proxy'} @pytest.mark.parametrize( 'url, expected, proxies', ( ('hTTp://u:p@Some.Host/path', 'http://some.host.proxy', http_proxies), @@ -336,6 +339,11 @@ all_proxies = {'all': 'socks5://http.proxy', ('hTTp:///path', 'socks5://http.proxy', all_proxies), ('hTTps://Other.Host', 'socks5://http.proxy', all_proxies), + ('http://u:p@other.host/path', 'http://http.proxy', mixed_proxies), + ('http://u:p@some.host/path', 'http://some.host.proxy', mixed_proxies), + ('https://u:p@other.host/path', 'socks5://http.proxy', mixed_proxies), + ('https://u:p@some.host/path', 'socks5://http.proxy', mixed_proxies), + # XXX: unsure whether this is reasonable behavior ('file:///etc/motd', 'socks5://http.proxy', all_proxies), )) From 7c80222afa17e3c02a41eb465f886699dfd56687 Mon Sep 17 00:00:00 2001 From: Michael Hunsinger Date: Mon, 22 Aug 2016 16:26:57 -0600 Subject: [PATCH 51/56] Fixed another scheme proxy over "all" priority --- requests/sessions.py | 2 +- requests/utils.py | 2 +- tests/test_utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index bcbcc880..38fa2880 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -226,7 +226,7 @@ class SessionRedirectMixin(object): if self.trust_env and not should_bypass_proxies(url): environ_proxies = get_environ_proxies(url) - proxy = environ_proxies.get('all', environ_proxies.get(scheme)) + proxy = environ_proxies.get(scheme, environ_proxies.get('all')) if proxy: new_proxies.setdefault(scheme, proxy) diff --git a/requests/utils.py b/requests/utils.py index e0bb3e7c..149b7a0d 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -617,7 +617,7 @@ def select_proxy(url, proxies): proxies = proxies or {} urlparts = urlparse(url) if urlparts.hostname is None: - return proxies.get('all', proxies.get(urlparts.scheme)) + return proxies.get(urlparts.scheme, proxies.get('all')) proxy_keys = [ urlparts.scheme + '://' + urlparts.hostname, diff --git a/tests/test_utils.py b/tests/test_utils.py index b4fc9483..03cff7a6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -343,7 +343,7 @@ mixed_proxies = {'http': 'http://http.proxy', ('http://u:p@some.host/path', 'http://some.host.proxy', mixed_proxies), ('https://u:p@other.host/path', 'socks5://http.proxy', mixed_proxies), ('https://u:p@some.host/path', 'socks5://http.proxy', mixed_proxies), - + ('https://', 'socks5://http.proxy', mixed_proxies), # XXX: unsure whether this is reasonable behavior ('file:///etc/motd', 'socks5://http.proxy', all_proxies), )) From 2cd05271be511162b9d0c10bfb310feb623bc927 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Fri, 26 Aug 2016 09:43:11 +0200 Subject: [PATCH 52/56] Add persistent examples --- docs/user/advanced.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 343ba0ec..7b82d4f6 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -208,6 +208,11 @@ You can pass ``verify`` the path to a CA_BUNDLE file or directory with certifica >>> requests.get('https://github.com', verify='/path/to/certfile') +or persistent:: + + s = requests.Session() + s.verify = '/path/to/certfile' + .. note:: If ``verify`` is set to a path to a directory, the directory must have been processed using the c_rehash utility supplied with OpenSSL. @@ -227,6 +232,11 @@ file's path:: >>> requests.get('https://kennethreitz.com', cert=('/path/client.cert', '/path/client.key')) +or persistent:: + + s = requests.Session() + s.cert = '/path/client.cert' + If you specify a wrong path or an invalid cert, you'll get a SSLError:: >>> requests.get('https://kennethreitz.com', cert='/wrong_path/client.pem') From 80f304fd30d926bbe9626c4a02146dcb4aa2e2e7 Mon Sep 17 00:00:00 2001 From: Richard van den Berg Date: Fri, 26 Aug 2016 15:17:31 +0200 Subject: [PATCH 53/56] Specify self.cert is used for SSL client certificates --- requests/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/sessions.py b/requests/sessions.py index 38fa2880..dc0388ff 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -322,7 +322,7 @@ class Session(SessionRedirectMixin): #: SSL Verification default. self.verify = True - #: SSL certificate default. + #: SSL client certificate default. self.cert = None #: Maximum number of redirects allowed. If the request exceeds this From cd056cd62109e1b6af16031cf9bc57cb5c38bbbf Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Fri, 26 Aug 2016 22:46:18 -0600 Subject: [PATCH 54/56] adding ISO-8859-1 fallback for reason decoding --- requests/models.py | 5 ++++- tests/test_requests.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 74385674..acde9e1b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -848,7 +848,10 @@ class Response(object): http_error_msg = '' if isinstance(self.reason, bytes): - reason = self.reason.decode('utf-8', 'ignore') + try: + reason = self.reason.decode('utf-8') + except UnicodeDecodeError: + reason = self.reason.decode('iso-8859-1') else: reason = self.reason diff --git a/tests/test_requests.py b/tests/test_requests.py index 6292f9d7..f85510f3 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1022,6 +1022,18 @@ class TestRequests: r.encoding = None assert not r.ok # old behaviour - crashes here + def test_response_reason_unicode_fallback(self): + # check raise_status falls back to ISO-8859-1 + r = requests.Response() + r.url = 'some url' + reason = u'Komponenttia ei löydy' + r.reason = reason.encode('latin-1') + r.status_code = 500 + r.encoding = None + with pytest.raises(requests.exceptions.HTTPError) as e: + r.raise_for_status() + assert reason in str(e) + def test_response_chunk_size_type(self): """Ensure that chunk_size is passed as None or an integer, otherwise raise a TypeError. From dedf9064c6fae669995d0dc04081f93845ca9c42 Mon Sep 17 00:00:00 2001 From: Ferhat Date: Mon, 5 Sep 2016 18:28:57 +0200 Subject: [PATCH 55/56] pysocks 1.5.7 blacklisting, due to IPv6 problems --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3a390522..bc991f3d 100755 --- a/setup.py +++ b/setup.py @@ -93,6 +93,6 @@ setup( tests_require=test_requirements, extras_require={ 'security': ['pyOpenSSL>=0.13', 'ndg-httpsclient', 'pyasn1'], - 'socks': ['PySocks>=1.5.6'], + 'socks': ['PySocks>=1.5.6, !=1.5.7'], }, ) From 2e1086f5b4fe1caf507abfb6cd2ac0c2ad420e26 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Mon, 5 Sep 2016 14:17:46 -0600 Subject: [PATCH 56/56] adding comment --- requests/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requests/models.py b/requests/models.py index acde9e1b..0f91b64e 100644 --- a/requests/models.py +++ b/requests/models.py @@ -848,6 +848,10 @@ class Response(object): http_error_msg = '' if isinstance(self.reason, bytes): + # We attempt to decode utf-8 first because some servers + # choose to localize their reason strings. If the string + # isn't utf-8, we fall back to iso-8859-1 for all other + # encodings. (See PR #3538) try: reason = self.reason.decode('utf-8') except UnicodeDecodeError: