diff --git a/HISTORY.rst b/HISTORY.rst index e56121df..8cd2b391 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,48 @@ Release History --------------- +2.7.0 (2015-05-03) +++++++++++++++++++ + +This is the first release that follows our new release process. For more, see +`our documentation +`_. + +**Bugfixes** + +- Updated urllib3 to 1.10.4, resolving several bugs involving chunked transfer + encoding and response framing. + +2.6.2 (2015-04-23) +++++++++++++++++++ + +**Bugfixes** + +- Fix regression where compressed data that was sent as chunked data was not + properly decompressed. (#2561) + +2.6.1 (2015-04-22) +++++++++++++++++++ + +**Bugfixes** + +- Remove VendorAlias import machinery introduced in v2.5.2. + +- Simplify the PreparedRequest.prepare API: We no longer require the user to + pass an empty list to the hooks keyword argument. (c.f. #2552) + +- Resolve redirects now receives and forwards all of the original arguments to + the adapter. (#2503) + +- Handle UnicodeDecodeErrors when trying to deal with a unicode URL that + cannot be encoded in ASCII. (#2540) + +- Populate the parsed path of the URI field when performing Digest + Authentication. (#2426) + +- Copy a PreparedRequest's CookieJar more reliably when it is not an instance + of RequestsCookieJar. (#2527) + 2.6.0 (2015-03-14) ++++++++++++++++++ diff --git a/docs/community/out-there.rst b/docs/community/out-there.rst index 235efd69..de41f1d4 100644 --- a/docs/community/out-there.rst +++ b/docs/community/out-there.rst @@ -30,3 +30,4 @@ Articles & Talks - `Issac Kelly's 'Consuming Web APIs' talk `_ - `Blog post about Requests via Yum `_ - `Russian blog post introducing Requests `_ +- `Sending JSON in Requests `_ diff --git a/docs/community/recommended.rst b/docs/community/recommended.rst index b7a550a0..99a16b9e 100644 --- a/docs/community/recommended.rst +++ b/docs/community/recommended.rst @@ -43,3 +43,15 @@ to provide authentication. It also provides a lot of tweaks that handle ways that specific OAuth providers differ from the standard specifications. .. _requests-oauthlib: https://requests-oauthlib.readthedocs.org/en/latest/ + + +Betamax +------- + +`Betamax`_ records your HTTP interactions so the NSA does not have to. +A VCR imitation designed only for Python-Requests. + +.. _betamax: https://github.com/sigmavirus24/betamax + + + diff --git a/docs/community/release-process.rst b/docs/community/release-process.rst new file mode 100644 index 00000000..adb86d5c --- /dev/null +++ b/docs/community/release-process.rst @@ -0,0 +1,54 @@ +Release Process and Rules +========================= + +.. versionadded:: v2.6.2 + +Starting with the version to be released after ``v2.6.2``, the following rules +will govern and describe how the Requests core team produces a new release. + +Major Releases +-------------- + +A major release will include breaking changes. When it is versioned, it will +be versioned as ``vX.0.0``. For example, if the previous release was +``v10.2.7`` the next version will be ``v11.0.0``. + +Breaking changes are changes that break backwards compatibility with prior +versions. If the project were to change the ``text`` attribute on a +``Response`` object to a method, that would only happen in a Major release. + +Major releases may also include miscellaneous bug fixes and upgrades to +vendored packages. The core developers of Requests are committed to providing +a good user experience. This means we're also committed to preserving +backwards compatibility as much as possible. Major releases will be infrequent +and will need strong justifications before they are considered. + +Minor Releases +-------------- + +A minor release will not include breaking changes but may include +miscellaneous bug fixes and upgrades to vendored packages. If the previous +version of Requests released was ``v10.2.7`` a minor release would be +versioned as ``v10.3.0``. + +Minor releases will be backwards compatible with releases that have the same +major version number. In other words, all versions that would start with +``v10.`` should be compatible with each other. + +Hotfix Releases +--------------- + +A hotfix release will only include bug fixes that were missed when the project +released the previous version. If the previous version of Requests released +``v10.2.7`` the hotfix release would be versioned as ``v10.2.8``. + +Hotfixes will **not** include upgrades to vendored dependences after +``v2.6.2`` + +Reasoning +--------- + +In the 2.5 and 2.6 release series, the Requests core team upgraded vendored +dependencies and caused a great deal of headaches for both users and the core +team. To reduce this pain, we're forming a concrete set of procedures so +expectations will be properly set. diff --git a/docs/community/updates.rst b/docs/community/updates.rst index 65e8c093..b4897a83 100644 --- a/docs/community/updates.rst +++ b/docs/community/updates.rst @@ -1,8 +1,6 @@ .. _updates: - - Community Updates ================= @@ -19,23 +17,14 @@ The best way to track the development of Requests is through Twitter ------- -I often tweet about new features and releases of Requests. +The author, Kenneth Reitz, often tweets about new features and releases of Requests. Follow `@kennethreitz `_ for updates. -Mailing List ------------- -There's a low-volume mailing list for Requests. To subscribe to the -mailing list, send an email to -`requests@librelist.org `_. - - - - -Software Updates -================ +Release and Version History +=========================== .. include:: ../../HISTORY.rst diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst new file mode 100644 index 00000000..53b2a8e5 --- /dev/null +++ b/docs/dev/contributing.rst @@ -0,0 +1,145 @@ +.. _contributing: + +Contributor's Guide +=================== + +If you're reading this you're probably interested in contributing to +Requests. First, We'd like to say: thank you! Open source projects +live-and-die based on the support they receive from others, and the fact that +you're even considering supporting Requests is very generous of +you. + +This document lays out guidelines and advice for contributing to Requests. +If you're thinking of contributing, start by reading this thoroughly and +getting a feel for how contributing to the project works. If you have any +questions, feel free to reach out to either `Ian Cordasco`_ or `Cory Benfield`_, +the primary maintainers. + +The guide is split into sections based on the type of contribution you're +thinking of making, with a section that covers general guidelines for all +contributors. + +.. _Ian Cordasco: http://www.coglib.com/~icordasc/ +.. _Cory Benfield: https://lukasa.co.uk/about + + +All Contributions +----------------- + +Be Cordial +~~~~~~~~~~ + +**Be cordial or be on your way.** + +Requests has one very important rule governing all forms of contribution, +including reporting bugs or requesting features. This golden rule is +`be cordial or be on your way`_. **All contributions are welcome**, as long as +everyone involved is treated with respect. + +.. _be cordial or be on your way: http://kennethreitz.org/be-cordial-or-be-on-your-way/ + +.. _early-feedback: + +Get Early Feedback +~~~~~~~~~~~~~~~~~~ + +If you are contributing, do not feel the need to sit on your contribution until +it is perfectly polished and complete. It helps everyone involved for you to +seek feedback as early as you possibly can. Submitting an early, unfinished +version of your contribution for feedback in no way prejudices your chances of +getting that contribution accepted, and can save you from putting a lot of work +into a contribution that is not suitable for the project. + +Contribution Suitability +~~~~~~~~~~~~~~~~~~~~~~~~ + +The project maintainer has the last word on whether or not a contribution is +suitable for Requests. All contributions will be considered, but from time +to time contributions will be rejected because they do not suit the project. + +If your contribution is rejected, don't despair! So long as you followed these +guidelines, you'll have a much better chance of getting your next contribution +accepted. + + +Code Contributions +------------------ + +Steps +~~~~~ + +When contributing code, you'll want to follow this checklist: + +1. Fork the repository on GitHub. +2. Run the tests to confirm they all pass on your system. If they don't, you'll + need to investigate why they fail. If you're unable to diagnose this + yourself, raise it as a bug report by following the guidelines in this + document: :ref:`bug-reports`. +3. Write tests that demonstrate your bug or feature. Ensure that they fail. +4. Make your change. +5. Run the entire test suite again, confirming that all tests pass *including + the ones you just added*. +6. Send a GitHub Pull Request to the main repository's ``master`` branch. + GitHub Pull Requests are the expected method of code collaboration on this + project. + +The following sub-sections go into more detail on some of the points above. + +Code Review +~~~~~~~~~~~ + +Contributions will not be merged until they've been code reviewed. You should +implement any code review feedback unless you strongly object to it. In the +event that you object to the code review feedback, you should make your case +clearly and calmly. If, after doing so, the feedback is judged to still apply, +you must either apply the feedback or withdraw your contribution. + +New Contributors +~~~~~~~~~~~~~~~~ + +If you are new or relatively new to Open Source, welcome! Requests aims to +be a gentle introduction to the world of Open Source. If you're concerned about +how best to contribute, please consider mailing a maintainer (listed above) and +asking for help. + +Please also check the :ref:`early-feedback` section. + +Documentation Contributions +--------------------------- + +Documentation improvements are always welcome! The documentation files live in +the ``docs/`` directory of the codebase. They're written in +`reStructuredText`_, and use `Sphinx`_ to generate the full suite of +documentation. + +When contributing documentation, please attempt to follow the style of the +documentation files. This means a soft-limit of 79 characters wide in your text +files and a semi-formal prose style. + +.. _reStructuredText: http://docutils.sourceforge.net/rst.html +.. _Sphinx: http://sphinx-doc.org/index.html + + +.. _bug-reports: + +Bug Reports +----------- + +Bug reports are hugely important! Before you raise one, though, please check +through the `GitHub issues`_, **both open and closed**, to confirm that the bug +hasn't been reported before. Duplicate bug reports are a huge drain on the time +of other contributors, and should be avoided as much as possible. + +.. _GitHub issues: https://github.com/kennethreitz/requests/issues + + +Feature Requests +---------------- + +Requests is in a perpeptual feature freeze. The maintainers believe that +requests contains every major feature currently required by the vast majority +of users. + +If you believe there is a feature missing, feel free to raise a feature +request, but please do be aware that the overwhelming likelihood is that your +feature request will not be accepted. diff --git a/docs/index.rst b/docs/index.rst index 12b24c9e..04ba7e50 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -115,10 +115,11 @@ Requests ecosystem and community. community/faq community/recommended - community/out-there.rst + community/out-there community/support community/vulnerabilities community/updates + community/release-process API Documentation ----------------- @@ -141,6 +142,7 @@ you. .. toctree:: :maxdepth: 1 + dev/contributing dev/philosophy dev/todo dev/authors diff --git a/requests/__init__.py b/requests/__init__.py index a403663b..d2471284 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -42,8 +42,8 @@ is at . """ __title__ = 'requests' -__version__ = '2.6.0' -__build__ = 0x020600 +__version__ = '2.7.0' +__build__ = 0x020700 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2015 Kenneth Reitz' diff --git a/requests/api.py b/requests/api.py index 98c92298..d40fa380 100644 --- a/requests/api.py +++ b/requests/api.py @@ -55,17 +55,18 @@ def request(method, url, **kwargs): return response -def get(url, **kwargs): +def get(url, params=None, **kwargs): """Sends a GET request. :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response """ kwargs.setdefault('allow_redirects', True) - return request('get', url, **kwargs) + return request('get', url, params=params, **kwargs) def options(url, **kwargs): diff --git a/requests/models.py b/requests/models.py index cb05e030..f32dd5f6 100644 --- a/requests/models.py +++ b/requests/models.py @@ -513,6 +513,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_hooks(self, hooks): """Prepares the given hooks.""" + # hooks can be passed as None to the prepare method and to this + # method. To prevent iterating over None, simply use an empty list + # if hooks is False-y + hooks = hooks or [] for event in hooks: self.register_hook(event, hooks[event]) diff --git a/requests/packages/__init__.py b/requests/packages/__init__.py index b73b4d1b..d62c4b71 100644 --- a/requests/packages/__init__.py +++ b/requests/packages/__init__.py @@ -1,107 +1,3 @@ -""" -Copyright (c) Donald Stufft, pip, and individual contributors - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" from __future__ import absolute_import -import sys - - -class VendorAlias(object): - - def __init__(self, package_names): - self._package_names = package_names - self._vendor_name = __name__ - self._vendor_pkg = self._vendor_name + "." - self._vendor_pkgs = [ - self._vendor_pkg + name for name in self._package_names - ] - - def find_module(self, fullname, path=None): - if fullname.startswith(self._vendor_pkg): - return self - - def load_module(self, name): - # Ensure that this only works for the vendored name - if not name.startswith(self._vendor_pkg): - raise ImportError( - "Cannot import %s, must be a subpackage of '%s'." % ( - name, self._vendor_name, - ) - ) - - if not (name == self._vendor_name or - any(name.startswith(pkg) for pkg in self._vendor_pkgs)): - raise ImportError( - "Cannot import %s, must be one of %s." % ( - name, self._vendor_pkgs - ) - ) - - # Check to see if we already have this item in sys.modules, if we do - # then simply return that. - if name in sys.modules: - return sys.modules[name] - - # Check to see if we can import the vendor name - try: - # We do this dance here because we want to try and import this - # module without hitting a recursion error because of a bunch of - # VendorAlias instances on sys.meta_path - real_meta_path = sys.meta_path[:] - try: - sys.meta_path = [ - m for m in sys.meta_path - if not isinstance(m, VendorAlias) - ] - __import__(name) - module = sys.modules[name] - finally: - # Re-add any additions to sys.meta_path that were made while - # during the import we just did, otherwise things like - # requests.packages.urllib3.poolmanager will fail. - for m in sys.meta_path: - if m not in real_meta_path: - real_meta_path.append(m) - - # Restore sys.meta_path with any new items. - sys.meta_path = real_meta_path - except ImportError: - # We can't import the vendor name, so we'll try to import the - # "real" name. - real_name = name[len(self._vendor_pkg):] - try: - __import__(real_name) - module = sys.modules[real_name] - except ImportError: - raise ImportError("No module named '%s'" % (name,)) - - # If we've gotten here we've found the module we're looking for, either - # as part of our vendored package, or as the real name, so we'll add - # it to sys.modules as the vendored name so that we don't have to do - # the lookup again. - sys.modules[name] = module - - # Finally, return the loaded module - return module - - -sys.meta_path.insert(0, VendorAlias(["urllib3", "chardet"])) +from . import urllib3 diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 0660b9c8..f48ac4af 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.10.2' +__version__ = '1.10.4' from .connectionpool import ( @@ -55,9 +55,12 @@ def add_stderr_logger(level=logging.DEBUG): del NullHandler -# Set security warning to always go off by default. import warnings -warnings.simplefilter('always', exceptions.SecurityWarning) +# SecurityWarning's always go off by default. +warnings.simplefilter('always', exceptions.SecurityWarning, append=True) +# InsecurePlatformWarning's don't vary between requests, so we keep it default. +warnings.simplefilter('default', exceptions.InsecurePlatformWarning, + append=True) def disable_warnings(category=exceptions.HTTPWarning): """ diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index cc424de0..279416ce 100644 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -227,20 +227,20 @@ class HTTPHeaderDict(dict): # Need to convert the tuple to list for further extension _dict_setitem(self, key_lower, [vals[0], vals[1], val]) - def extend(*args, **kwargs): + def extend(self, *args, **kwargs): """Generic import function for any type of header-like object. Adapted version of MutableMapping.update in order to insert items with self.add instead of self.__setitem__ """ - if len(args) > 2: - raise TypeError("update() takes at most 2 positional " + if len(args) > 1: + raise TypeError("extend() takes at most 1 positional " "arguments ({} given)".format(len(args))) - elif not args: - raise TypeError("update() takes at least 1 argument (0 given)") - self = args[0] - other = args[1] if len(args) >= 2 else () + other = args[0] if len(args) >= 1 else () - if isinstance(other, Mapping): + if isinstance(other, HTTPHeaderDict): + for key, val in other.iteritems(): + self.add(key, val) + elif isinstance(other, Mapping): for key in other: self.add(key, other[key]) elif hasattr(other, "keys"): @@ -304,17 +304,20 @@ class HTTPHeaderDict(dict): return list(self.iteritems()) @classmethod - def from_httplib(cls, message, duplicates=('set-cookie',)): # Python 2 + def from_httplib(cls, message): # Python 2 """Read headers from a Python 2 httplib message object.""" - ret = cls(message.items()) - # ret now contains only the last header line for each duplicate. - # Importing with all duplicates would be nice, but this would - # mean to repeat most of the raw parsing already done, when the - # message object was created. Extracting only the headers of interest - # separately, the cookies, should be faster and requires less - # extra code. - for key in duplicates: - ret.discard(key) - for val in message.getheaders(key): - ret.add(key, val) - return ret + # python2.7 does not expose a proper API for exporting multiheaders + # efficiently. This function re-reads raw lines from the message + # object and extracts the multiheaders properly. + headers = [] + + for line in message.headers: + if line.startswith((' ', '\t')): + key, value = headers[-1] + headers[-1] = (key, value + '\r\n' + line.rstrip()) + continue + + key, value = line.split(':', 1) + headers.append((key, value.strip())) + + return cls(headers) diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py index e5de769d..2a8c3596 100644 --- a/requests/packages/urllib3/connection.py +++ b/requests/packages/urllib3/connection.py @@ -260,3 +260,5 @@ if ssl: # Make a copy for testing. UnverifiedHTTPSConnection = HTTPSConnection HTTPSConnection = VerifiedHTTPSConnection +else: + HTTPSConnection = DummyConnection diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 0085345c..117269ac 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -735,7 +735,6 @@ class HTTPSConnectionPool(HTTPConnectionPool): % (self.num_connections, self.host)) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: - # Platform-specific: Python without ssl raise SSLError("Can't connect to HTTPS URL because the SSL " "module is not available.") diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py index ee657fb3..b2c34a89 100644 --- a/requests/packages/urllib3/contrib/pyopenssl.py +++ b/requests/packages/urllib3/contrib/pyopenssl.py @@ -38,8 +38,6 @@ Module Variables ---------------- :var DEFAULT_SSL_CIPHER_LIST: The list of supported SSL/TLS cipher suites. - Default: ``ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES: - ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS`` .. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication .. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) @@ -85,22 +83,7 @@ _openssl_verify = { + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } -# A secure default. -# Sources for more information on TLS ciphers: -# -# - https://wiki.mozilla.org/Security/Server_Side_TLS -# - https://www.ssllabs.com/projects/best-practices/index.html -# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ -# -# The general intent is: -# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), -# - prefer ECDHE over DHE for better performance, -# - prefer any AES-GCM over any AES-CBC for better performance and security, -# - use 3DES as fallback which is secure but slow, -# - disable NULL authentication, MD5 MACs and DSS for security reasons. -DEFAULT_SSL_CIPHER_LIST = "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:" + \ - "ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:" + \ - "!aNULL:!MD5:!DSS" +DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS orig_util_HAS_SNI = util.HAS_SNI @@ -299,7 +282,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, try: cnx.do_handshake() except OpenSSL.SSL.WantReadError: - select.select([sock], [], []) + rd, _, _ = select.select([sock], [], [], sock.gettimeout()) + if not rd: + raise timeout('select timed out') continue except OpenSSL.SSL.Error as e: raise ssl.SSLError('bad handshake', e) diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 5d523011..31bda1c0 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -162,3 +162,8 @@ class SystemTimeWarning(SecurityWarning): class InsecurePlatformWarning(SecurityWarning): "Warned when certain SSL configuration is not available on a platform." pass + + +class ResponseNotChunked(ProtocolError, ValueError): + "Response needs to be chunked in order to read it as chunks." + pass diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 34cd3d70..24140c4c 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -1,9 +1,15 @@ +try: + import http.client as httplib +except ImportError: + import httplib import zlib import io from socket import timeout as SocketTimeout from ._collections import HTTPHeaderDict -from .exceptions import ProtocolError, DecodeError, ReadTimeoutError +from .exceptions import ( + ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked +) from .packages.six import string_types as basestring, binary_type, PY3 from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed @@ -117,7 +123,17 @@ class HTTPResponse(io.IOBase): if hasattr(body, 'read'): self._fp = body - if preload_content and not self._body: + # Are we using the chunked-style of transfer encoding? + self.chunked = False + self.chunk_left = None + tr_enc = self.headers.get('transfer-encoding', '').lower() + # Don't incur the penalty of creating a list and then discarding it + encodings = (enc.strip() for enc in tr_enc.split(",")) + if "chunked" in encodings: + self.chunked = True + + # We certainly don't want to preload content when the response is chunked. + if not self.chunked and preload_content and not self._body: self._body = self.read(decode_content=decode_content) def get_redirect_location(self): @@ -157,6 +173,35 @@ class HTTPResponse(io.IOBase): """ return self._fp_bytes_read + def _init_decoder(self): + """ + Set-up the _decoder attribute if necessar. + """ + # Note: content-encoding value should be case-insensitive, per RFC 7230 + # Section 3.2 + content_encoding = self.headers.get('content-encoding', '').lower() + if self._decoder is None and content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + + def _decode(self, data, decode_content, flush_decoder): + """ + Decode the data passed in and potentially flush the decoder. + """ + try: + if decode_content and self._decoder: + data = self._decoder.decompress(data) + except (IOError, zlib.error) as e: + content_encoding = self.headers.get('content-encoding', '').lower() + raise DecodeError( + "Received response with content-encoding: %s, but " + "failed to decode it." % content_encoding, e) + + if flush_decoder and decode_content and self._decoder: + buf = self._decoder.decompress(binary_type()) + data += buf + self._decoder.flush() + + return data + def read(self, amt=None, decode_content=None, cache_content=False): """ Similar to :meth:`httplib.HTTPResponse.read`, but with two additional @@ -178,12 +223,7 @@ class HTTPResponse(io.IOBase): after having ``.read()`` the file object. (Overridden if ``amt`` is set.) """ - # Note: content-encoding value should be case-insensitive, per RFC 7230 - # Section 3.2 - content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None: - if content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + self._init_decoder() if decode_content is None: decode_content = self.decode_content @@ -232,17 +272,7 @@ class HTTPResponse(io.IOBase): self._fp_bytes_read += len(data) - try: - if decode_content and self._decoder: - data = self._decoder.decompress(data) - except (IOError, zlib.error) as e: - raise DecodeError( - "Received response with content-encoding: %s, but " - "failed to decode it." % content_encoding, e) - - if flush_decoder and decode_content and self._decoder: - buf = self._decoder.decompress(binary_type()) - data += buf + self._decoder.flush() + data = self._decode(data, decode_content, flush_decoder) if cache_content: self._body = data @@ -269,11 +299,15 @@ class HTTPResponse(io.IOBase): If True, will attempt to decode the body based on the 'content-encoding' header. """ - while not is_fp_closed(self._fp): - data = self.read(amt=amt, decode_content=decode_content) + if self.chunked: + for line in self.read_chunked(amt, decode_content=decode_content): + yield line + else: + while not is_fp_closed(self._fp): + data = self.read(amt=amt, decode_content=decode_content) - if data: - yield data + if data: + yield data @classmethod def from_httplib(ResponseCls, r, **response_kw): @@ -351,3 +385,82 @@ class HTTPResponse(io.IOBase): else: b[:len(temp)] = temp return len(temp) + + def _update_chunk_length(self): + # First, we'll figure out length of a chunk and then + # we'll try to read it from socket. + if self.chunk_left is not None: + return + line = self._fp.fp.readline() + line = line.split(b';', 1)[0] + try: + self.chunk_left = int(line, 16) + except ValueError: + # Invalid chunked protocol response, abort. + self.close() + raise httplib.IncompleteRead(line) + + def _handle_chunk(self, amt): + returned_chunk = None + if amt is None: + chunk = self._fp._safe_read(self.chunk_left) + returned_chunk = chunk + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + elif amt < self.chunk_left: + value = self._fp._safe_read(amt) + self.chunk_left = self.chunk_left - amt + returned_chunk = value + elif amt == self.chunk_left: + value = self._fp._safe_read(amt) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + returned_chunk = value + else: # amt > self.chunk_left + returned_chunk = self._fp._safe_read(self.chunk_left) + self._fp._safe_read(2) # Toss the CRLF at the end of the chunk. + self.chunk_left = None + return returned_chunk + + def read_chunked(self, amt=None, decode_content=None): + """ + Similar to :meth:`HTTPResponse.read`, but with an additional + parameter: ``decode_content``. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + self._init_decoder() + # FIXME: Rewrite this method and make it a class with a better structured logic. + if not self.chunked: + raise ResponseNotChunked("Response is not chunked. " + "Header 'transfer-encoding: chunked' is missing.") + + if self._original_response and self._original_response._method.upper() == 'HEAD': + # Don't bother reading the body of a HEAD request. + # FIXME: Can we do this somehow without accessing private httplib _method? + self._original_response.close() + return + + while True: + self._update_chunk_length() + if self.chunk_left == 0: + break + chunk = self._handle_chunk(amt) + yield self._decode(chunk, decode_content=decode_content, + flush_decoder=True) + + # Chunk content ends with \r\n: discard it. + while True: + line = self._fp.fp.readline() + if not line: + # Some sites may not end with '\r\n'. + break + if line == b'\r\n': + break + + # We read everything; close the "file". + if self._original_response: + self._original_response.close() + self.release_conn() diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py index e7e7dfae..b846d42c 100644 --- a/requests/packages/urllib3/util/ssl_.py +++ b/requests/packages/urllib3/util/ssl_.py @@ -9,10 +9,10 @@ HAS_SNI = False create_default_context = None import errno -import ssl import warnings try: # Test for SSL features + import ssl from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 from ssl import HAS_SNI # Has SNI? except ImportError: @@ -25,14 +25,24 @@ except ImportError: OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 OP_NO_COMPRESSION = 0x20000 -try: - from ssl import _DEFAULT_CIPHERS -except ImportError: - _DEFAULT_CIPHERS = ( - 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:' - 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:' - '!eNULL:!MD5' - ) +# A secure default. +# Sources for more information on TLS ciphers: +# +# - https://wiki.mozilla.org/Security/Server_Side_TLS +# - https://www.ssllabs.com/projects/best-practices/index.html +# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ +# +# The general intent is: +# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), +# - prefer ECDHE over DHE for better performance, +# - prefer any AES-GCM over any AES-CBC for better performance and security, +# - use 3DES as fallback which is secure but slow, +# - disable NULL authentication, MD5 MACs and DSS for security reasons. +DEFAULT_CIPHERS = ( + 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:' + 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:' + '!eNULL:!MD5' +) try: from ssl import SSLContext # Modern SSL? @@ -40,7 +50,8 @@ except ImportError: import sys class SSLContext(object): # Platform-specific: Python 2 & 3.1 - supports_set_ciphers = sys.version_info >= (2, 7) + supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or + (3, 2) <= sys.version_info) def __init__(self, protocol_version): self.protocol = protocol_version @@ -167,7 +178,7 @@ def resolve_ssl_version(candidate): return candidate -def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED, +def create_urllib3_context(ssl_version=None, cert_reqs=None, options=None, ciphers=None): """All arguments have the same meaning as ``ssl_wrap_socket``. @@ -204,6 +215,9 @@ def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED, """ context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + # Setting the default here, as we may have no ssl module on import + cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs + if options is None: options = 0 # SSLv2 is easily broken and is considered harmful and dangerous @@ -217,7 +231,7 @@ def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED, context.options |= options if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 - context.set_ciphers(ciphers or _DEFAULT_CIPHERS) + context.set_ciphers(ciphers or DEFAULT_CIPHERS) context.verify_mode = cert_reqs if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 diff --git a/requests/packages/urllib3/util/url.py b/requests/packages/urllib3/util/url.py index b2ec834f..e58050cd 100644 --- a/requests/packages/urllib3/util/url.py +++ b/requests/packages/urllib3/util/url.py @@ -15,6 +15,8 @@ class Url(namedtuple('Url', url_attrs)): def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None): + if path and not path.startswith('/'): + path = '/' + path return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment) diff --git a/test_requests.py b/test_requests.py index 15406a22..cad8c055 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1613,7 +1613,6 @@ def test_prepare_unicode_url(): p.prepare( method='GET', url=u('http://www.example.com/üniçø∂é'), - hooks=[] ) assert_copy(p, p.copy())