From 35045e851a75b23d3d4e50853dfc1c5555110895 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 20 Apr 2015 20:13:57 -0500 Subject: [PATCH 01/22] Simplify PreparedRequest.prepare API Do not require that hooks be passed as an empty list to PreparedRequest.prepare. In the event hooks is None in prepare or prepare_hooks, use an empty list as a default. Related to #2552 --- requests/models.py | 4 ++++ test_requests.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 752c58c1..45b3ea96 100644 --- a/requests/models.py +++ b/requests/models.py @@ -523,6 +523,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_hooks(self, hooks): """Prepares the given hooks.""" + # hooks can be passed as None to the prepare method and to this + # method. To prevent iterating over None, simply use an empty list + # if hooks is False-y + hooks = hooks or [] for event in hooks: self.register_hook(event, hooks[event]) diff --git a/test_requests.py b/test_requests.py index 15406a22..cad8c055 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1613,7 +1613,6 @@ def test_prepare_unicode_url(): p.prepare( method='GET', url=u('http://www.example.com/üniçø∂é'), - hooks=[] ) assert_copy(p, p.copy()) From 4cfc5bcd965ed060bf109edad28c89c93c0f8562 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 21 Apr 2015 17:15:51 -0400 Subject: [PATCH 02/22] add @sigmavirus42's blog post --- docs/community/out-there.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/community/out-there.rst b/docs/community/out-there.rst index 235efd69..de41f1d4 100644 --- a/docs/community/out-there.rst +++ b/docs/community/out-there.rst @@ -30,3 +30,4 @@ Articles & Talks - `Issac Kelly's 'Consuming Web APIs' talk `_ - `Blog post about Requests via Yum `_ - `Russian blog post introducing Requests `_ +- `Sending JSON in Requests `_ From 1f0e5775f28d6c2db3430fdc16a095762816bf87 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 21 Apr 2015 17:18:04 -0400 Subject: [PATCH 03/22] add betamax --- docs/community/recommended.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/community/recommended.rst b/docs/community/recommended.rst index b7a550a0..99a16b9e 100644 --- a/docs/community/recommended.rst +++ b/docs/community/recommended.rst @@ -43,3 +43,15 @@ to provide authentication. It also provides a lot of tweaks that handle ways that specific OAuth providers differ from the standard specifications. .. _requests-oauthlib: https://requests-oauthlib.readthedocs.org/en/latest/ + + +Betamax +------- + +`Betamax`_ records your HTTP interactions so the NSA does not have to. +A VCR imitation designed only for Python-Requests. + +.. _betamax: https://github.com/sigmavirus24/betamax + + + From 5fcd843eb23e8e84cd5f60c6d372ef4d678f80fe Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 22 Apr 2015 08:17:39 -0500 Subject: [PATCH 04/22] Update urllib3 to 10b7a0fefa6596f47a9a6afc80f1f4d1ae950b66 --- requests/packages/urllib3/__init__.py | 4 +- requests/packages/urllib3/_collections.py | 45 +++++----- requests/packages/urllib3/connection.py | 2 + requests/packages/urllib3/connectionpool.py | 1 - .../packages/urllib3/contrib/pyopenssl.py | 23 +---- requests/packages/urllib3/exceptions.py | 5 ++ requests/packages/urllib3/response.py | 89 +++++++++++++++++-- requests/packages/urllib3/util/ssl_.py | 38 +++++--- 8 files changed, 146 insertions(+), 61 deletions(-) diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 0660b9c8..8cd51444 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -55,9 +55,11 @@ def add_stderr_logger(level=logging.DEBUG): del NullHandler -# Set security warning to always go off by default. import warnings +# SecurityWarning's always go off by default. warnings.simplefilter('always', exceptions.SecurityWarning) +# InsecurePlatformWarning's don't vary between requests, so we keep it default. +warnings.simplefilter('default', exceptions.InsecurePlatformWarning) def disable_warnings(category=exceptions.HTTPWarning): """ diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index cc424de0..279416ce 100644 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -227,20 +227,20 @@ class HTTPHeaderDict(dict): # Need to convert the tuple to list for further extension _dict_setitem(self, key_lower, [vals[0], vals[1], val]) - def extend(*args, **kwargs): + def extend(self, *args, **kwargs): """Generic import function for any type of header-like object. Adapted version of MutableMapping.update in order to insert items with self.add instead of self.__setitem__ """ - if len(args) > 2: - raise TypeError("update() takes at most 2 positional " + if len(args) > 1: + raise TypeError("extend() takes at most 1 positional " "arguments ({} given)".format(len(args))) - elif not args: - raise TypeError("update() takes at least 1 argument (0 given)") - self = args[0] - other = args[1] if len(args) >= 2 else () + other = args[0] if len(args) >= 1 else () - if isinstance(other, Mapping): + if isinstance(other, HTTPHeaderDict): + for key, val in other.iteritems(): + self.add(key, val) + elif isinstance(other, Mapping): for key in other: self.add(key, other[key]) elif hasattr(other, "keys"): @@ -304,17 +304,20 @@ class HTTPHeaderDict(dict): return list(self.iteritems()) @classmethod - def from_httplib(cls, message, duplicates=('set-cookie',)): # Python 2 + def from_httplib(cls, message): # Python 2 """Read headers from a Python 2 httplib message object.""" - ret = cls(message.items()) - # ret now contains only the last header line for each duplicate. - # Importing with all duplicates would be nice, but this would - # mean to repeat most of the raw parsing already done, when the - # message object was created. Extracting only the headers of interest - # separately, the cookies, should be faster and requires less - # extra code. - for key in duplicates: - ret.discard(key) - for val in message.getheaders(key): - ret.add(key, val) - return ret + # python2.7 does not expose a proper API for exporting multiheaders + # efficiently. This function re-reads raw lines from the message + # object and extracts the multiheaders properly. + headers = [] + + for line in message.headers: + if line.startswith((' ', '\t')): + key, value = headers[-1] + headers[-1] = (key, value + '\r\n' + line.rstrip()) + continue + + key, value = line.split(':', 1) + headers.append((key, value.strip())) + + return cls(headers) diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py index e5de769d..2a8c3596 100644 --- a/requests/packages/urllib3/connection.py +++ b/requests/packages/urllib3/connection.py @@ -260,3 +260,5 @@ if ssl: # Make a copy for testing. UnverifiedHTTPSConnection = HTTPSConnection HTTPSConnection = VerifiedHTTPSConnection +else: + HTTPSConnection = DummyConnection diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 0085345c..117269ac 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -735,7 +735,6 @@ class HTTPSConnectionPool(HTTPConnectionPool): % (self.num_connections, self.host)) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: - # Platform-specific: Python without ssl raise SSLError("Can't connect to HTTPS URL because the SSL " "module is not available.") diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py index ee657fb3..b2c34a89 100644 --- a/requests/packages/urllib3/contrib/pyopenssl.py +++ b/requests/packages/urllib3/contrib/pyopenssl.py @@ -38,8 +38,6 @@ Module Variables ---------------- :var DEFAULT_SSL_CIPHER_LIST: The list of supported SSL/TLS cipher suites. - Default: ``ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES: - ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS`` .. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication .. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) @@ -85,22 +83,7 @@ _openssl_verify = { + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } -# A secure default. -# Sources for more information on TLS ciphers: -# -# - https://wiki.mozilla.org/Security/Server_Side_TLS -# - https://www.ssllabs.com/projects/best-practices/index.html -# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ -# -# The general intent is: -# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), -# - prefer ECDHE over DHE for better performance, -# - prefer any AES-GCM over any AES-CBC for better performance and security, -# - use 3DES as fallback which is secure but slow, -# - disable NULL authentication, MD5 MACs and DSS for security reasons. -DEFAULT_SSL_CIPHER_LIST = "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:" + \ - "ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:" + \ - "!aNULL:!MD5:!DSS" +DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS orig_util_HAS_SNI = util.HAS_SNI @@ -299,7 +282,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, try: cnx.do_handshake() except OpenSSL.SSL.WantReadError: - select.select([sock], [], []) + rd, _, _ = select.select([sock], [], [], sock.gettimeout()) + if not rd: + raise timeout('select timed out') continue except OpenSSL.SSL.Error as e: raise ssl.SSLError('bad handshake', e) diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 5d523011..31bda1c0 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -162,3 +162,8 @@ class SystemTimeWarning(SecurityWarning): class InsecurePlatformWarning(SecurityWarning): "Warned when certain SSL configuration is not available on a platform." pass + + +class ResponseNotChunked(ProtocolError, ValueError): + "Response needs to be chunked in order to read it as chunks." + pass diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 34cd3d70..7e08fffe 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -1,9 +1,15 @@ +try: + import http.client as httplib +except ImportError: + import httplib import zlib import io from socket import timeout as SocketTimeout from ._collections import HTTPHeaderDict -from .exceptions import ProtocolError, DecodeError, ReadTimeoutError +from .exceptions import ( + ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked +) from .packages.six import string_types as basestring, binary_type, PY3 from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed @@ -117,8 +123,17 @@ class HTTPResponse(io.IOBase): if hasattr(body, 'read'): self._fp = body - if preload_content and not self._body: - self._body = self.read(decode_content=decode_content) + # Are we using the chunked-style of transfer encoding? + self.chunked = False + self.chunk_left = None + tr_enc = self.headers.get('transfer-encoding', '') + if tr_enc.lower() == "chunked": + self.chunked = True + + # We certainly don't want to preload content when the response is chunked. + if not self.chunked: + if preload_content and not self._body: + self._body = self.read(decode_content=decode_content) def get_redirect_location(self): """ @@ -269,11 +284,15 @@ class HTTPResponse(io.IOBase): If True, will attempt to decode the body based on the 'content-encoding' header. """ - while not is_fp_closed(self._fp): - data = self.read(amt=amt, decode_content=decode_content) + if self.chunked: + for line in self.read_chunked(amt): + yield line + else: + while not is_fp_closed(self._fp): + data = self.read(amt=amt, decode_content=decode_content) - if data: - yield data + if data: + yield data @classmethod def from_httplib(ResponseCls, r, **response_kw): @@ -351,3 +370,59 @@ class HTTPResponse(io.IOBase): else: b[:len(temp)] = temp return len(temp) + + def read_chunked(self, amt=None): + # FIXME: Rewrite this method and make it a class with + # a better structured logic. + if not self.chunked: + raise ResponseNotChunked("Response is not chunked. " + "Header 'transfer-encoding: chunked' is missing.") + while True: + # First, we'll figure out length of a chunk and then + # we'll try to read it from socket. + if self.chunk_left is None: + line = self._fp.fp.readline() + line = line.decode() + # See RFC 7230: Chunked Transfer Coding. + i = line.find(';') + if i >= 0: + line = line[:i] # Strip chunk-extensions. + try: + self.chunk_left = int(line, 16) + except ValueError: + # Invalid chunked protocol response, abort. + self.close() + raise httplib.IncompleteRead(''.join(line)) + if self.chunk_left == 0: + break + if amt is None: + chunk = self._fp._safe_read(self.chunk_left) + yield chunk + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + elif amt < self.chunk_left: + value = self._fp._safe_read(amt) + self.chunk_left = self.chunk_left - amt + yield value + elif amt == self.chunk_left: + value = self._fp._safe_read(amt) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + yield value + else: # amt > self.chunk_left + yield self._fp._safe_read(self.chunk_left) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + + # Chunk content ends with \r\n: discard it. + while True: + line = self._fp.fp.readline() + if not line: + # Some sites may not end with '\r\n'. + break + if line == b'\r\n': + break + + # We read everything; close the "file". + self.release_conn() + diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py index e7e7dfae..b846d42c 100644 --- a/requests/packages/urllib3/util/ssl_.py +++ b/requests/packages/urllib3/util/ssl_.py @@ -9,10 +9,10 @@ HAS_SNI = False create_default_context = None import errno -import ssl import warnings try: # Test for SSL features + import ssl from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 from ssl import HAS_SNI # Has SNI? except ImportError: @@ -25,14 +25,24 @@ except ImportError: OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 OP_NO_COMPRESSION = 0x20000 -try: - from ssl import _DEFAULT_CIPHERS -except ImportError: - _DEFAULT_CIPHERS = ( - 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:' - 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:' - '!eNULL:!MD5' - ) +# A secure default. +# Sources for more information on TLS ciphers: +# +# - https://wiki.mozilla.org/Security/Server_Side_TLS +# - https://www.ssllabs.com/projects/best-practices/index.html +# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ +# +# The general intent is: +# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), +# - prefer ECDHE over DHE for better performance, +# - prefer any AES-GCM over any AES-CBC for better performance and security, +# - use 3DES as fallback which is secure but slow, +# - disable NULL authentication, MD5 MACs and DSS for security reasons. +DEFAULT_CIPHERS = ( + 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:' + 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:' + '!eNULL:!MD5' +) try: from ssl import SSLContext # Modern SSL? @@ -40,7 +50,8 @@ except ImportError: import sys class SSLContext(object): # Platform-specific: Python 2 & 3.1 - supports_set_ciphers = sys.version_info >= (2, 7) + supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or + (3, 2) <= sys.version_info) def __init__(self, protocol_version): self.protocol = protocol_version @@ -167,7 +178,7 @@ def resolve_ssl_version(candidate): return candidate -def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED, +def create_urllib3_context(ssl_version=None, cert_reqs=None, options=None, ciphers=None): """All arguments have the same meaning as ``ssl_wrap_socket``. @@ -204,6 +215,9 @@ def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED, """ context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + # Setting the default here, as we may have no ssl module on import + cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs + if options is None: options = 0 # SSLv2 is easily broken and is considered harmful and dangerous @@ -217,7 +231,7 @@ def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED, context.options |= options if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 - context.set_ciphers(ciphers or _DEFAULT_CIPHERS) + context.set_ciphers(ciphers or DEFAULT_CIPHERS) context.verify_mode = cert_reqs if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 From cfb9e66b04136f014412a8d638b2d69603557c1d Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 22 Apr 2015 08:43:20 -0500 Subject: [PATCH 05/22] Update history and version number for v2.6.1 --- HISTORY.rst | 23 +++++++++++++++++++++++ requests/__init__.py | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e56121df..19391150 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,29 @@ Release History --------------- +2.6.1 (2015-04-22) +++++++++++++++++++ + +**Bugfixes** + +- Fix issue with import machinery to deal with an unvendored urllib3 and + chardet when operating from within a PyInstaller executable. (#2465 redux) + +- Simplify the PreparedRequest.prepare API: We no longer require the user to + pass an empty list to the hooks keyword argument. (c.f. #2552) + +- Resolve redirects now receives and forwards all of the original arguments to + the adapter. (#2503) + +- Handle UnicodeDecodeErrors when trying to deal with a unicode URL that + cannot be encoded in ASCII. (#2540) + +- Populate the parsed path of the URI field when performing Digest + Authentication. (#2426) + +- Copy a PreparedRequest's CookieJar more reliably when it is not an instance + of RequestsCookieJar. (#2527) + 2.6.0 (2015-03-14) ++++++++++++++++++ diff --git a/requests/__init__.py b/requests/__init__.py index 47165761..4174629d 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -42,8 +42,8 @@ is at . """ __title__ = 'requests' -__version__ = '2.6.0' -__build__ = 0x020600 +__version__ = '2.6.1' +__build__ = 0x020601 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2015 Kenneth Reitz' From 3d339ed7e61261832347190185113b890b55009e Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 22 Apr 2015 18:39:40 +0100 Subject: [PATCH 06/22] Update urllib3 to 0b744993 (cherry picked from commit df63ee3e13672caa6b3256c6579fa85caefb2729) --- requests/packages/urllib3/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 8cd51444..333060c2 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.10.2' +__version__ = '1.10.3' from .connectionpool import ( From 3e3fc7683cab2d401981fdf01ae8cdc40c8d6222 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 22 Apr 2015 16:26:14 -0500 Subject: [PATCH 07/22] Remove VendorAlias meta_path hook --- requests/packages/__init__.py | 106 +--------------------------------- 1 file changed, 1 insertion(+), 105 deletions(-) diff --git a/requests/packages/__init__.py b/requests/packages/__init__.py index b73b4d1b..d62c4b71 100644 --- a/requests/packages/__init__.py +++ b/requests/packages/__init__.py @@ -1,107 +1,3 @@ -""" -Copyright (c) Donald Stufft, pip, and individual contributors - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" from __future__ import absolute_import -import sys - - -class VendorAlias(object): - - def __init__(self, package_names): - self._package_names = package_names - self._vendor_name = __name__ - self._vendor_pkg = self._vendor_name + "." - self._vendor_pkgs = [ - self._vendor_pkg + name for name in self._package_names - ] - - def find_module(self, fullname, path=None): - if fullname.startswith(self._vendor_pkg): - return self - - def load_module(self, name): - # Ensure that this only works for the vendored name - if not name.startswith(self._vendor_pkg): - raise ImportError( - "Cannot import %s, must be a subpackage of '%s'." % ( - name, self._vendor_name, - ) - ) - - if not (name == self._vendor_name or - any(name.startswith(pkg) for pkg in self._vendor_pkgs)): - raise ImportError( - "Cannot import %s, must be one of %s." % ( - name, self._vendor_pkgs - ) - ) - - # Check to see if we already have this item in sys.modules, if we do - # then simply return that. - if name in sys.modules: - return sys.modules[name] - - # Check to see if we can import the vendor name - try: - # We do this dance here because we want to try and import this - # module without hitting a recursion error because of a bunch of - # VendorAlias instances on sys.meta_path - real_meta_path = sys.meta_path[:] - try: - sys.meta_path = [ - m for m in sys.meta_path - if not isinstance(m, VendorAlias) - ] - __import__(name) - module = sys.modules[name] - finally: - # Re-add any additions to sys.meta_path that were made while - # during the import we just did, otherwise things like - # requests.packages.urllib3.poolmanager will fail. - for m in sys.meta_path: - if m not in real_meta_path: - real_meta_path.append(m) - - # Restore sys.meta_path with any new items. - sys.meta_path = real_meta_path - except ImportError: - # We can't import the vendor name, so we'll try to import the - # "real" name. - real_name = name[len(self._vendor_pkg):] - try: - __import__(real_name) - module = sys.modules[real_name] - except ImportError: - raise ImportError("No module named '%s'" % (name,)) - - # If we've gotten here we've found the module we're looking for, either - # as part of our vendored package, or as the real name, so we'll add - # it to sys.modules as the vendored name so that we don't have to do - # the lookup again. - sys.modules[name] = module - - # Finally, return the loaded module - return module - - -sys.meta_path.insert(0, VendorAlias(["urllib3", "chardet"])) +from . import urllib3 From ae7c28eb8e8df89555b50d766b77dc922c1500ce Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 22 Apr 2015 16:52:40 -0500 Subject: [PATCH 08/22] Update notes for 2.6.1 --- HISTORY.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 19391150..d0967b7d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,8 +8,7 @@ Release History **Bugfixes** -- Fix issue with import machinery to deal with an unvendored urllib3 and - chardet when operating from within a PyInstaller executable. (#2465 redux) +- Remove VendorAlias import machinery introduced in v2.5.2. - Simplify the PreparedRequest.prepare API: We no longer require the user to pass an empty list to the hooks keyword argument. (c.f. #2552) From 73b9b6906d5cd94bd18751d7c5dce15a5d47eab0 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 23 Apr 2015 11:27:42 -0500 Subject: [PATCH 09/22] Update urllib3 to 7b82da0fa3a13514d11a863e379e7541d0ec46ef --- requests/packages/urllib3/response.py | 52 +++++++++++++++++---------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 7e08fffe..f1ea9bb5 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -172,6 +172,36 @@ class HTTPResponse(io.IOBase): """ return self._fp_bytes_read + def _init_decoder(self): + """ + Set-up the _decoder attribute if necessar. + """ + # Note: content-encoding value should be case-insensitive, per RFC 7230 + # Section 3.2 + content_encoding = self.headers.get('content-encoding', '').lower() + if self._decoder is None: + if content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + + def _decode(self, data, decode_content, flush_decoder): + """ + Decode the data passed in and potentially flush the decoder. + """ + try: + if decode_content and self._decoder: + data = self._decoder.decompress(data) + except (IOError, zlib.error) as e: + content_encoding = self.headers.get('content-encoding', '').lower() + raise DecodeError( + "Received response with content-encoding: %s, but " + "failed to decode it." % content_encoding, e) + + if flush_decoder and decode_content and self._decoder: + buf = self._decoder.decompress(binary_type()) + data += buf + self._decoder.flush() + + return data + def read(self, amt=None, decode_content=None, cache_content=False): """ Similar to :meth:`httplib.HTTPResponse.read`, but with two additional @@ -193,12 +223,7 @@ class HTTPResponse(io.IOBase): after having ``.read()`` the file object. (Overridden if ``amt`` is set.) """ - # Note: content-encoding value should be case-insensitive, per RFC 7230 - # Section 3.2 - content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None: - if content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + self._init_decoder() if decode_content is None: decode_content = self.decode_content @@ -247,17 +272,7 @@ class HTTPResponse(io.IOBase): self._fp_bytes_read += len(data) - try: - if decode_content and self._decoder: - data = self._decoder.decompress(data) - except (IOError, zlib.error) as e: - raise DecodeError( - "Received response with content-encoding: %s, but " - "failed to decode it." % content_encoding, e) - - if flush_decoder and decode_content and self._decoder: - buf = self._decoder.decompress(binary_type()) - data += buf + self._decoder.flush() + data = self._decode(data, decode_content, flush_decoder) if cache_content: self._body = data @@ -284,9 +299,10 @@ class HTTPResponse(io.IOBase): If True, will attempt to decode the body based on the 'content-encoding' header. """ + self._init_decoder() if self.chunked: for line in self.read_chunked(amt): - yield line + yield self._decode(line, decode_content, True) else: while not is_fp_closed(self._fp): data = self.read(amt=amt, decode_content=decode_content) From ca66267d2cf8adc67ed8857f3f57b45ffde21e01 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 23 Apr 2015 11:29:10 -0500 Subject: [PATCH 10/22] Add release notes for 2.6.2 Closes #2561 --- HISTORY.rst | 8 ++++++++ requests/__init__.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d0967b7d..815a1734 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,14 @@ Release History --------------- +2.6.2 (2015-04-23) +++++++++++++++++++ + +**Bugfixes** + +- Fix regression where compressed data that was sent as chunked data was not + properly decompressed. (#2561) + 2.6.1 (2015-04-22) ++++++++++++++++++ diff --git a/requests/__init__.py b/requests/__init__.py index 4174629d..46116bea 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -42,8 +42,8 @@ is at . """ __title__ = 'requests' -__version__ = '2.6.1' -__build__ = 0x020601 +__version__ = '2.6.2' +__build__ = 0x020602 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2015 Kenneth Reitz' From 773e730d80aa5e590ca0fb0fa079a9b4f11d6403 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 24 Apr 2015 19:28:01 -0500 Subject: [PATCH 11/22] Add new release process to docs Closes #2569 --- docs/community/release-process.rst | 51 ++++++++++++++++++++++++++++++ docs/index.rst | 3 +- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 docs/community/release-process.rst diff --git a/docs/community/release-process.rst b/docs/community/release-process.rst new file mode 100644 index 00000000..5fad3f46 --- /dev/null +++ b/docs/community/release-process.rst @@ -0,0 +1,51 @@ +Release Process and Rules +========================= + +.. versionadded:: v2.6.2 + +Starting with the version to be released after ``v2.6.2``, the following rules +will govern and describe how the Requests core team produces a new release. + +Major Releases +-------------- + +A major release will include breaking changes. When it is versioned, it will +be versioned as ``vX.0.0``. For example, if the previous release was +``v10.2.9`` the next version will be ``v11.0.0``. + +Breaking changes are changes that break backwards compatibility with prior +versions. If the project were to change the ``text`` attribute on a +``Response`` object to a method, that would only happen in a Major release. + +Major releases may also include miscellaneous bug fixes and upgrades to +vendored packages. + +Minor Releases +-------------- + +A minor release will not include breaking changes but may include +miscellaneous bug fixes and upgrades to vendored packages. If the previous +version of Requests released was ``v10.2.9`` a minor release would be +versioned as ``v10.3.0``. + +Minor releases will be backwards compatible with releases that have the same +major version number. In other words, all versions that would start with +``v10.`` should be compatible with each other. + +Hotfix Releases +--------------- + +A hotfix release will only include bug fixes that were missed when the project +released the previous version. If the previous version of Requests released +``v10.2.9`` the hotfix release would be versioned as ``v10.2.10``. + +Hotfixes will **not** include upgrades to vendored dependences after +``v2.6.2`` + +Reasoning +--------- + +In the 2.5 and 2.6 release series, the Requests core team upgraded vendored +dependencies and caused a great deal of headaches for both users and the core +team. To reduce this pain, we're forming a concrete set of procedures so +expectations will be properly set. diff --git a/docs/index.rst b/docs/index.rst index 12b24c9e..e5d07d53 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -115,10 +115,11 @@ Requests ecosystem and community. community/faq community/recommended - community/out-there.rst + community/out-there community/support community/vulnerabilities community/updates + community/release-process API Documentation ----------------- From f74fc486be3ba16914a46df809ca33fbffb28629 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 25 Apr 2015 09:29:40 -0500 Subject: [PATCH 12/22] Update version number examples. Strip whitespace Also I've included a bit more verbiage around why a major release would happen and some reassurances that they will be rare. --- docs/community/release-process.rst | 43 ++++++++++++++++-------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/docs/community/release-process.rst b/docs/community/release-process.rst index 5fad3f46..adb86d5c 100644 --- a/docs/community/release-process.rst +++ b/docs/community/release-process.rst @@ -3,49 +3,52 @@ Release Process and Rules .. versionadded:: v2.6.2 -Starting with the version to be released after ``v2.6.2``, the following rules +Starting with the version to be released after ``v2.6.2``, the following rules will govern and describe how the Requests core team produces a new release. Major Releases -------------- -A major release will include breaking changes. When it is versioned, it will -be versioned as ``vX.0.0``. For example, if the previous release was -``v10.2.9`` the next version will be ``v11.0.0``. +A major release will include breaking changes. When it is versioned, it will +be versioned as ``vX.0.0``. For example, if the previous release was +``v10.2.7`` the next version will be ``v11.0.0``. -Breaking changes are changes that break backwards compatibility with prior -versions. If the project were to change the ``text`` attribute on a +Breaking changes are changes that break backwards compatibility with prior +versions. If the project were to change the ``text`` attribute on a ``Response`` object to a method, that would only happen in a Major release. -Major releases may also include miscellaneous bug fixes and upgrades to -vendored packages. +Major releases may also include miscellaneous bug fixes and upgrades to +vendored packages. The core developers of Requests are committed to providing +a good user experience. This means we're also committed to preserving +backwards compatibility as much as possible. Major releases will be infrequent +and will need strong justifications before they are considered. Minor Releases -------------- -A minor release will not include breaking changes but may include -miscellaneous bug fixes and upgrades to vendored packages. If the previous -version of Requests released was ``v10.2.9`` a minor release would be +A minor release will not include breaking changes but may include +miscellaneous bug fixes and upgrades to vendored packages. If the previous +version of Requests released was ``v10.2.7`` a minor release would be versioned as ``v10.3.0``. -Minor releases will be backwards compatible with releases that have the same -major version number. In other words, all versions that would start with +Minor releases will be backwards compatible with releases that have the same +major version number. In other words, all versions that would start with ``v10.`` should be compatible with each other. Hotfix Releases --------------- -A hotfix release will only include bug fixes that were missed when the project -released the previous version. If the previous version of Requests released -``v10.2.9`` the hotfix release would be versioned as ``v10.2.10``. +A hotfix release will only include bug fixes that were missed when the project +released the previous version. If the previous version of Requests released +``v10.2.7`` the hotfix release would be versioned as ``v10.2.8``. -Hotfixes will **not** include upgrades to vendored dependences after +Hotfixes will **not** include upgrades to vendored dependences after ``v2.6.2`` Reasoning --------- -In the 2.5 and 2.6 release series, the Requests core team upgraded vendored -dependencies and caused a great deal of headaches for both users and the core -team. To reduce this pain, we're forming a concrete set of procedures so +In the 2.5 and 2.6 release series, the Requests core team upgraded vendored +dependencies and caused a great deal of headaches for both users and the core +team. To reduce this pain, we're forming a concrete set of procedures so expectations will be properly set. From 9dba20395296f26a56fde851cbfa8ac6cef5a3bf Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Sat, 25 Apr 2015 21:39:00 +0100 Subject: [PATCH 13/22] Add basic contributing guide --- docs/dev/contributing.rst | 161 ++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 162 insertions(+) create mode 100644 docs/dev/contributing.rst diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst new file mode 100644 index 00000000..472010cc --- /dev/null +++ b/docs/dev/contributing.rst @@ -0,0 +1,161 @@ +.. _contributing: + +Contributor's Guide +=================== + +If you're reading this you're probably interested in contributing to +``requests``. First, I'd like to say: thankyou! Open source projects +live-and-die based on the support they receive from others, and the fact that +you're even *considering* supporting ``requests`` is incredibly generous of +you. + +This document lays out guidelines and advice for contributing to ``requests``. +If you're thinking of contributing, start by reading this thoroughly and +getting a feel for how contributing to the project works. If you've still got +questions after reading this, you should go ahead and contact either +`Ian Cordasco`_ or `Cory Benfield`_, the active maintainers. + +The guide is split into sections based on the type of contribution you're +thinking of making, with a section that covers general guidelines for all +contributors. + +.. _Ian Cordasco: http://www.coglib.com/~icordasc/ +.. _Cory Benfield: https://lukasa.co.uk/about + + +All Contributions +----------------- + +Be Cordial Or Be On Your Way +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``requests`` has one very important guideline governing all forms of +contribution, including things like reporting bugs or requesting features. The +guideline is `be cordial or be on your way`_. **All contributions are +welcome**, but they come with an implicit social contract: everyone must be +treated with respect. + +This can be a difficult area to judge, so the maintainers will enforce the +following policy. If any contributor acts rudely or aggressively towards any +other contributor, **regardless of whether they perceive themselves to be +acting in retaliation for an earlier breach of this guideline**, they will be +subject to the following procedure: + +1. They must apologise. This apology must be genuine in nature: "I'm sorry you + were offended" is not sufficient. The judgement of 'genuine' is at the + discretion of the maintainers. +2. If the apology is not offered, any outstanding and future contributions from + the violating contributor will be rejected immediately. + +Everyone involved in the ``requests`` project, the maintainers included, are +bound by this policy. Failing to abide by it leads to the offender being kicked +off the project. + +.. _be cordial or be on your way: http://kennethreitz.org/be-cordial-or-be-on-your-way/ + +.. _early-feedback: + +Get Early Feedback +~~~~~~~~~~~~~~~~~~ + +If you are contributing, do not feel the need to sit on your contribution until +it is perfectly polished and complete. It helps everyone involved for you to +seek feedback as early as you possibly can. Submitting an early, unfinished +version of your contribution for feedback in no way prejudices your chances of +getting that contribution accepted, and can save you from putting a lot of work +into a contribution that is not suitable for the project. + +Contribution Suitability +~~~~~~~~~~~~~~~~~~~~~~~~ + +The project maintainer has the last word on whether or not a contribution is +suitable for ``requests``. All contributions will be considered, but from time +to time contributions will be rejected because they do not suit the project. + +If your contribution is rejected, don't despair! So long as you followed these +guidelines, you'll have a much better chance of getting your next contribution +accepted. + + +Code Contributions +------------------ + +Steps +~~~~~ + +When contributing code, you'll want to follow this checklist: + +1. Fork the repository on GitHub. +2. Run the tests to confirm they all pass on your system. If they don't, you'll + need to investigate why they fail. If you're unable to diagnose this + yourself, raise it as a bug report by following the guidelines in this + document: :ref:`bug-reports`. +3. Write tests that demonstrate your bug or feature. Ensure that they fail. +4. Make your change. +5. Run the entire test suite again, confirming that all tests pass *including + the ones you just added*. +6. Send a GitHub Pull Request to the main repository's ``master`` branch. + GitHub Pull Requests are the expected method of code collaboration on this + project. If you object to the GitHub workflow, you may mail a patch to any + of the maintainers listed above. + +The following sub-sections go into more detail on some of the points above. + +Code Review +~~~~~~~~~~~ + +Contributions will not be merged until they've been code reviewed. You should +implement any code review feedback unless you strongly object to it. In the +event that you object to the code review feedback, you should make your case +clearly and calmly. If, after doing so, the feedback is judged to still apply, +you must either apply the feedback or withdraw your contribution. + +New Contributors +~~~~~~~~~~~~~~~~ + +If you are new or relatively new to Open Source, welcome! ``requests`` aims to +be a gentle introduction to the world of Open Source. If you're concerned about +how best to contribute, please consider mailing a maintainer (listed above) and +asking for help. + +Please also check the :ref:`early-feedback` section. + +Documentation Contributions +--------------------------- + +Documentation improvements are always welcome! The documentation files live in +the ``docs/`` directory of the codebase. They're written in +`reStructuredText`_, and use `Sphinx`_ to generate the full suite of +documentation. + +When contributing documentation, please attempt to follow the style of the +documentation files. This means a soft-limit of 79 characters wide in your text +files and a semi-formal prose style. + +.. _reStructuredText: http://docutils.sourceforge.net/rst.html +.. _Sphinx: http://sphinx-doc.org/index.html + + +.. _bug-reports: + +Bug Reports +----------- + +Bug reports are hugely important! Before you raise one, though, please check +through the `GitHub issues`_, **both open and closed**, to confirm that the bug +hasn't been reported before. Duplicate bug reports are a huge drain on the time +of other contributors, and should be avoided as much as possible. + +.. _GitHub issues: https://github.com/kennethreitz/requests/issues + + +Feature Requests +---------------- + +Requests is in a perpeptual feature freeze. The maintainers believe that +requests contains every major feature currently required by the vast majority +of users. + +If you believe there is a feature missing, feel free to raise a feature +request, but please do be aware that the overwhelming likelihood is that your +feature request will not be accepted. diff --git a/docs/index.rst b/docs/index.rst index 12b24c9e..fb6a8457 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -141,6 +141,7 @@ you. .. toctree:: :maxdepth: 1 + dev/contributing dev/philosophy dev/todo dev/authors From 535c51f5481ab6276ac43dc73264a6d24aefec0f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 25 Apr 2015 17:36:14 -0400 Subject: [PATCH 14/22] style of contributing.rst --- docs/dev/contributing.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 472010cc..115464ad 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -4,12 +4,12 @@ Contributor's Guide =================== If you're reading this you're probably interested in contributing to -``requests``. First, I'd like to say: thankyou! Open source projects +``Requests``. First, I'd like to say: thankyou! Open source projects live-and-die based on the support they receive from others, and the fact that -you're even *considering* supporting ``requests`` is incredibly generous of +you're even *considering* supporting ``Requests`` is incredibly generous of you. -This document lays out guidelines and advice for contributing to ``requests``. +This document lays out guidelines and advice for contributing to ``Requests``. If you're thinking of contributing, start by reading this thoroughly and getting a feel for how contributing to the project works. If you've still got questions after reading this, you should go ahead and contact either @@ -29,7 +29,7 @@ All Contributions Be Cordial Or Be On Your Way ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``requests`` has one very important guideline governing all forms of +``Requests`` has one very important guideline governing all forms of contribution, including things like reporting bugs or requesting features. The guideline is `be cordial or be on your way`_. **All contributions are welcome**, but they come with an implicit social contract: everyone must be @@ -47,7 +47,7 @@ subject to the following procedure: 2. If the apology is not offered, any outstanding and future contributions from the violating contributor will be rejected immediately. -Everyone involved in the ``requests`` project, the maintainers included, are +Everyone involved in the ``Requests`` project, the maintainers included, are bound by this policy. Failing to abide by it leads to the offender being kicked off the project. @@ -69,7 +69,7 @@ Contribution Suitability ~~~~~~~~~~~~~~~~~~~~~~~~ The project maintainer has the last word on whether or not a contribution is -suitable for ``requests``. All contributions will be considered, but from time +suitable for ``Requests``. All contributions will be considered, but from time to time contributions will be rejected because they do not suit the project. If your contribution is rejected, don't despair! So long as you followed these @@ -113,7 +113,7 @@ you must either apply the feedback or withdraw your contribution. New Contributors ~~~~~~~~~~~~~~~~ -If you are new or relatively new to Open Source, welcome! ``requests`` aims to +If you are new or relatively new to Open Source, welcome! ``Requests`` aims to be a gentle introduction to the world of Open Source. If you're concerned about how best to contribute, please consider mailing a maintainer (listed above) and asking for help. From 51938de6b74cdf15b000bd8da76467784f8293e6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 25 Apr 2015 17:42:48 -0400 Subject: [PATCH 15/22] improve contributing section --- docs/dev/contributing.rst | 44 ++++++++++++--------------------------- 1 file changed, 13 insertions(+), 31 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 115464ad..572e408a 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -4,16 +4,16 @@ Contributor's Guide =================== If you're reading this you're probably interested in contributing to -``Requests``. First, I'd like to say: thankyou! Open source projects +Requests. First, We'd like to say: thank you! Open source projects live-and-die based on the support they receive from others, and the fact that -you're even *considering* supporting ``Requests`` is incredibly generous of +you're even considering supporting Requests is very generous of you. -This document lays out guidelines and advice for contributing to ``Requests``. +This document lays out guidelines and advice for contributing to Requests. If you're thinking of contributing, start by reading this thoroughly and -getting a feel for how contributing to the project works. If you've still got -questions after reading this, you should go ahead and contact either -`Ian Cordasco`_ or `Cory Benfield`_, the active maintainers. +getting a feel for how contributing to the project works. If you have any +questions, feel free to reach out to either `Ian Cordasco`_ or `Cory Benfield`_, +the primary maintainers. The guide is split into sections based on the type of contribution you're thinking of making, with a section that covers general guidelines for all @@ -29,27 +29,10 @@ All Contributions Be Cordial Or Be On Your Way ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``Requests`` has one very important guideline governing all forms of -contribution, including things like reporting bugs or requesting features. The -guideline is `be cordial or be on your way`_. **All contributions are -welcome**, but they come with an implicit social contract: everyone must be -treated with respect. - -This can be a difficult area to judge, so the maintainers will enforce the -following policy. If any contributor acts rudely or aggressively towards any -other contributor, **regardless of whether they perceive themselves to be -acting in retaliation for an earlier breach of this guideline**, they will be -subject to the following procedure: - -1. They must apologise. This apology must be genuine in nature: "I'm sorry you - were offended" is not sufficient. The judgement of 'genuine' is at the - discretion of the maintainers. -2. If the apology is not offered, any outstanding and future contributions from - the violating contributor will be rejected immediately. - -Everyone involved in the ``Requests`` project, the maintainers included, are -bound by this policy. Failing to abide by it leads to the offender being kicked -off the project. +Requests has one very important rule governing all forms of contribution, +including reporting bugs or requesting features. This golden rule is +`be cordial or be on your way`_. **All contributions are welcome**, as long as +everyone involved is treated with respect. .. _be cordial or be on your way: http://kennethreitz.org/be-cordial-or-be-on-your-way/ @@ -69,7 +52,7 @@ Contribution Suitability ~~~~~~~~~~~~~~~~~~~~~~~~ The project maintainer has the last word on whether or not a contribution is -suitable for ``Requests``. All contributions will be considered, but from time +suitable for Requests. All contributions will be considered, but from time to time contributions will be rejected because they do not suit the project. If your contribution is rejected, don't despair! So long as you followed these @@ -96,8 +79,7 @@ When contributing code, you'll want to follow this checklist: the ones you just added*. 6. Send a GitHub Pull Request to the main repository's ``master`` branch. GitHub Pull Requests are the expected method of code collaboration on this - project. If you object to the GitHub workflow, you may mail a patch to any - of the maintainers listed above. + project. The following sub-sections go into more detail on some of the points above. @@ -113,7 +95,7 @@ you must either apply the feedback or withdraw your contribution. New Contributors ~~~~~~~~~~~~~~~~ -If you are new or relatively new to Open Source, welcome! ``Requests`` aims to +If you are new or relatively new to Open Source, welcome! Requests aims to be a gentle introduction to the world of Open Source. If you're concerned about how best to contribute, please consider mailing a maintainer (listed above) and asking for help. From 1f85ce86700130e17e553b943e85048345d36041 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 25 Apr 2015 17:44:25 -0400 Subject: [PATCH 16/22] be cordial --- docs/dev/contributing.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 572e408a..53b2a8e5 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -26,8 +26,10 @@ contributors. All Contributions ----------------- -Be Cordial Or Be On Your Way -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Be Cordial +~~~~~~~~~~ + +**Be cordial or be on your way.** Requests has one very important rule governing all forms of contribution, including reporting bugs or requesting features. This golden rule is From e85920455485389a99719184ae4fee4848352328 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 25 Apr 2015 17:49:12 -0400 Subject: [PATCH 17/22] new community --- docs/community/updates.rst | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/docs/community/updates.rst b/docs/community/updates.rst index 65e8c093..e19b48db 100644 --- a/docs/community/updates.rst +++ b/docs/community/updates.rst @@ -19,21 +19,11 @@ The best way to track the development of Requests is through Twitter ------- -I often tweet about new features and releases of Requests. +The author, Kenneth Reitz, often tweets about new features and releases of Requests. Follow `@kennethreitz `_ for updates. -Mailing List ------------- - -There's a low-volume mailing list for Requests. To subscribe to the -mailing list, send an email to -`requests@librelist.org `_. - - - - Software Updates ================ From ff71b25efc44460d941b97a021a2b58bc3e0ef2e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 25 Apr 2015 17:50:03 -0400 Subject: [PATCH 18/22] release and version history --- docs/community/updates.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/community/updates.rst b/docs/community/updates.rst index e19b48db..b4897a83 100644 --- a/docs/community/updates.rst +++ b/docs/community/updates.rst @@ -1,8 +1,6 @@ .. _updates: - - Community Updates ================= @@ -24,8 +22,9 @@ The author, Kenneth Reitz, often tweets about new features and releases of Reque Follow `@kennethreitz `_ for updates. -Software Updates -================ + +Release and Version History +=========================== .. include:: ../../HISTORY.rst From 6e9ebc06d1056d5a488e8d9af75ae82f458bf714 Mon Sep 17 00:00:00 2001 From: Matt Havard Date: Thu, 30 Apr 2015 23:54:34 -0400 Subject: [PATCH 19/22] Make docstring of api.get more clear --- requests/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requests/api.py b/requests/api.py index 98c92298..d40fa380 100644 --- a/requests/api.py +++ b/requests/api.py @@ -55,17 +55,18 @@ def request(method, url, **kwargs): return response -def get(url, **kwargs): +def get(url, params=None, **kwargs): """Sends a GET request. :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response """ kwargs.setdefault('allow_redirects', True) - return request('get', url, **kwargs) + return request('get', url, params=params, **kwargs) def options(url, **kwargs): From ee7389da98092f6e63f68a490e7f71651c9ec047 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Sun, 3 May 2015 15:02:40 +0100 Subject: [PATCH 20/22] Synchronize urllib3 to 1.10.4 --- requests/packages/urllib3/__init__.py | 7 +- requests/packages/urllib3/response.py | 122 +++++++++++++++----------- requests/packages/urllib3/util/url.py | 2 + 3 files changed, 78 insertions(+), 53 deletions(-) diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 333060c2..f48ac4af 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.10.3' +__version__ = '1.10.4' from .connectionpool import ( @@ -57,9 +57,10 @@ del NullHandler import warnings # SecurityWarning's always go off by default. -warnings.simplefilter('always', exceptions.SecurityWarning) +warnings.simplefilter('always', exceptions.SecurityWarning, append=True) # InsecurePlatformWarning's don't vary between requests, so we keep it default. -warnings.simplefilter('default', exceptions.InsecurePlatformWarning) +warnings.simplefilter('default', exceptions.InsecurePlatformWarning, + append=True) def disable_warnings(category=exceptions.HTTPWarning): """ diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index f1ea9bb5..24140c4c 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -126,14 +126,15 @@ class HTTPResponse(io.IOBase): # Are we using the chunked-style of transfer encoding? self.chunked = False self.chunk_left = None - tr_enc = self.headers.get('transfer-encoding', '') - if tr_enc.lower() == "chunked": + tr_enc = self.headers.get('transfer-encoding', '').lower() + # Don't incur the penalty of creating a list and then discarding it + encodings = (enc.strip() for enc in tr_enc.split(",")) + if "chunked" in encodings: self.chunked = True # We certainly don't want to preload content when the response is chunked. - if not self.chunked: - if preload_content and not self._body: - self._body = self.read(decode_content=decode_content) + if not self.chunked and preload_content and not self._body: + self._body = self.read(decode_content=decode_content) def get_redirect_location(self): """ @@ -179,9 +180,8 @@ class HTTPResponse(io.IOBase): # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None: - if content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + if self._decoder is None and content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) def _decode(self, data, decode_content, flush_decoder): """ @@ -299,10 +299,9 @@ class HTTPResponse(io.IOBase): If True, will attempt to decode the body based on the 'content-encoding' header. """ - self._init_decoder() if self.chunked: - for line in self.read_chunked(amt): - yield self._decode(line, decode_content, True) + for line in self.read_chunked(amt, decode_content=decode_content): + yield line else: while not is_fp_closed(self._fp): data = self.read(amt=amt, decode_content=decode_content) @@ -387,48 +386,70 @@ class HTTPResponse(io.IOBase): b[:len(temp)] = temp return len(temp) - def read_chunked(self, amt=None): - # FIXME: Rewrite this method and make it a class with - # a better structured logic. + def _update_chunk_length(self): + # First, we'll figure out length of a chunk and then + # we'll try to read it from socket. + if self.chunk_left is not None: + return + line = self._fp.fp.readline() + line = line.split(b';', 1)[0] + try: + self.chunk_left = int(line, 16) + except ValueError: + # Invalid chunked protocol response, abort. + self.close() + raise httplib.IncompleteRead(line) + + def _handle_chunk(self, amt): + returned_chunk = None + if amt is None: + chunk = self._fp._safe_read(self.chunk_left) + returned_chunk = chunk + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + elif amt < self.chunk_left: + value = self._fp._safe_read(amt) + self.chunk_left = self.chunk_left - amt + returned_chunk = value + elif amt == self.chunk_left: + value = self._fp._safe_read(amt) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + returned_chunk = value + else: # amt > self.chunk_left + returned_chunk = self._fp._safe_read(self.chunk_left) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + return returned_chunk + + def read_chunked(self, amt=None, decode_content=None): + """ + Similar to :meth:`HTTPResponse.read`, but with an additional + parameter: ``decode_content``. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + self._init_decoder() + # FIXME: Rewrite this method and make it a class with a better structured logic. if not self.chunked: raise ResponseNotChunked("Response is not chunked. " "Header 'transfer-encoding: chunked' is missing.") + + if self._original_response and self._original_response._method.upper() == 'HEAD': + # Don't bother reading the body of a HEAD request. + # FIXME: Can we do this somehow without accessing private httplib _method? + self._original_response.close() + return + while True: - # First, we'll figure out length of a chunk and then - # we'll try to read it from socket. - if self.chunk_left is None: - line = self._fp.fp.readline() - line = line.decode() - # See RFC 7230: Chunked Transfer Coding. - i = line.find(';') - if i >= 0: - line = line[:i] # Strip chunk-extensions. - try: - self.chunk_left = int(line, 16) - except ValueError: - # Invalid chunked protocol response, abort. - self.close() - raise httplib.IncompleteRead(''.join(line)) - if self.chunk_left == 0: - break - if amt is None: - chunk = self._fp._safe_read(self.chunk_left) - yield chunk - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. - self.chunk_left = None - elif amt < self.chunk_left: - value = self._fp._safe_read(amt) - self.chunk_left = self.chunk_left - amt - yield value - elif amt == self.chunk_left: - value = self._fp._safe_read(amt) - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. - self.chunk_left = None - yield value - else: # amt > self.chunk_left - yield self._fp._safe_read(self.chunk_left) - self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. - self.chunk_left = None + self._update_chunk_length() + if self.chunk_left == 0: + break + chunk = self._handle_chunk(amt) + yield self._decode(chunk, decode_content=decode_content, + flush_decoder=True) # Chunk content ends with \r\n: discard it. while True: @@ -440,5 +461,6 @@ class HTTPResponse(io.IOBase): break # We read everything; close the "file". + if self._original_response: + self._original_response.close() self.release_conn() - diff --git a/requests/packages/urllib3/util/url.py b/requests/packages/urllib3/util/url.py index b2ec834f..e58050cd 100644 --- a/requests/packages/urllib3/util/url.py +++ b/requests/packages/urllib3/util/url.py @@ -15,6 +15,8 @@ class Url(namedtuple('Url', url_attrs)): def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None): + if path and not path.startswith('/'): + path = '/' + path return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment) From b7bd29734022e634634d908e10bbf7a5ac15e76b Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Sun, 3 May 2015 15:45:47 +0100 Subject: [PATCH 21/22] v2.7.0 --- HISTORY.rst | 11 +++++++++++ requests/__init__.py | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 815a1734..09446b34 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,17 @@ Release History --------------- +2.7.0 (2015-05-03) +++++++++++++++++++ + +This is the first release that follows our new release process. For more, see +[our documentation](http://docs.python-requests.org/en/latest/community/release-process/). + +**Bugfixes** + +- Updated urllib3 to 1.10.4, resolving several bugs involving chunked transfer + encoding and response framing. + 2.6.2 (2015-04-23) ++++++++++++++++++ diff --git a/requests/__init__.py b/requests/__init__.py index 46116bea..4bca66e7 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -42,8 +42,8 @@ is at . """ __title__ = 'requests' -__version__ = '2.6.2' -__build__ = 0x020602 +__version__ = '2.7.0' +__build__ = 0x020700 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2015 Kenneth Reitz' From ba7975fd165d8511c3c83039a65deffa7341d4ca Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 3 May 2015 20:34:09 -0400 Subject: [PATCH 22/22] use reST link syntax --- HISTORY.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 09446b34..8cd2b391 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,7 +7,8 @@ Release History ++++++++++++++++++ This is the first release that follows our new release process. For more, see -[our documentation](http://docs.python-requests.org/en/latest/community/release-process/). +`our documentation +`_. **Bugfixes**