diff --git a/HISTORY.rst b/HISTORY.rst
index e56121df..8cd2b391 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,6 +3,48 @@
Release History
---------------
+2.7.0 (2015-05-03)
+++++++++++++++++++
+
+This is the first release that follows our new release process. For more, see
+`our documentation
+`_.
+
+**Bugfixes**
+
+- Updated urllib3 to 1.10.4, resolving several bugs involving chunked transfer
+ encoding and response framing.
+
+2.6.2 (2015-04-23)
+++++++++++++++++++
+
+**Bugfixes**
+
+- Fix regression where compressed data that was sent as chunked data was not
+ properly decompressed. (#2561)
+
+2.6.1 (2015-04-22)
+++++++++++++++++++
+
+**Bugfixes**
+
+- Remove VendorAlias import machinery introduced in v2.5.2.
+
+- Simplify the PreparedRequest.prepare API: We no longer require the user to
+ pass an empty list to the hooks keyword argument. (c.f. #2552)
+
+- Resolve redirects now receives and forwards all of the original arguments to
+ the adapter. (#2503)
+
+- Handle UnicodeDecodeErrors when trying to deal with a unicode URL that
+ cannot be encoded in ASCII. (#2540)
+
+- Populate the parsed path of the URI field when performing Digest
+ Authentication. (#2426)
+
+- Copy a PreparedRequest's CookieJar more reliably when it is not an instance
+ of RequestsCookieJar. (#2527)
+
2.6.0 (2015-03-14)
++++++++++++++++++
diff --git a/docs/community/out-there.rst b/docs/community/out-there.rst
index 235efd69..de41f1d4 100644
--- a/docs/community/out-there.rst
+++ b/docs/community/out-there.rst
@@ -30,3 +30,4 @@ Articles & Talks
- `Issac Kelly's 'Consuming Web APIs' talk `_
- `Blog post about Requests via Yum `_
- `Russian blog post introducing Requests `_
+- `Sending JSON in Requests `_
diff --git a/docs/community/recommended.rst b/docs/community/recommended.rst
index b7a550a0..99a16b9e 100644
--- a/docs/community/recommended.rst
+++ b/docs/community/recommended.rst
@@ -43,3 +43,15 @@ to provide authentication. It also provides a lot of tweaks that handle ways
that specific OAuth providers differ from the standard specifications.
.. _requests-oauthlib: https://requests-oauthlib.readthedocs.org/en/latest/
+
+
+Betamax
+-------
+
+`Betamax`_ records your HTTP interactions so the NSA does not have to.
+A VCR imitation designed only for Python-Requests.
+
+.. _betamax: https://github.com/sigmavirus24/betamax
+
+
+
diff --git a/docs/community/release-process.rst b/docs/community/release-process.rst
new file mode 100644
index 00000000..adb86d5c
--- /dev/null
+++ b/docs/community/release-process.rst
@@ -0,0 +1,54 @@
+Release Process and Rules
+=========================
+
+.. versionadded:: v2.6.2
+
+Starting with the version to be released after ``v2.6.2``, the following rules
+will govern and describe how the Requests core team produces a new release.
+
+Major Releases
+--------------
+
+A major release will include breaking changes. When it is versioned, it will
+be versioned as ``vX.0.0``. For example, if the previous release was
+``v10.2.7`` the next version will be ``v11.0.0``.
+
+Breaking changes are changes that break backwards compatibility with prior
+versions. If the project were to change the ``text`` attribute on a
+``Response`` object to a method, that would only happen in a Major release.
+
+Major releases may also include miscellaneous bug fixes and upgrades to
+vendored packages. The core developers of Requests are committed to providing
+a good user experience. This means we're also committed to preserving
+backwards compatibility as much as possible. Major releases will be infrequent
+and will need strong justifications before they are considered.
+
+Minor Releases
+--------------
+
+A minor release will not include breaking changes but may include
+miscellaneous bug fixes and upgrades to vendored packages. If the previous
+version of Requests released was ``v10.2.7`` a minor release would be
+versioned as ``v10.3.0``.
+
+Minor releases will be backwards compatible with releases that have the same
+major version number. In other words, all versions that would start with
+``v10.`` should be compatible with each other.
+
+Hotfix Releases
+---------------
+
+A hotfix release will only include bug fixes that were missed when the project
+released the previous version. If the previous version of Requests released
+``v10.2.7`` the hotfix release would be versioned as ``v10.2.8``.
+
+Hotfixes will **not** include upgrades to vendored dependences after
+``v2.6.2``
+
+Reasoning
+---------
+
+In the 2.5 and 2.6 release series, the Requests core team upgraded vendored
+dependencies and caused a great deal of headaches for both users and the core
+team. To reduce this pain, we're forming a concrete set of procedures so
+expectations will be properly set.
diff --git a/docs/community/updates.rst b/docs/community/updates.rst
index 65e8c093..b4897a83 100644
--- a/docs/community/updates.rst
+++ b/docs/community/updates.rst
@@ -1,8 +1,6 @@
.. _updates:
-
-
Community Updates
=================
@@ -19,23 +17,14 @@ The best way to track the development of Requests is through
Twitter
-------
-I often tweet about new features and releases of Requests.
+The author, Kenneth Reitz, often tweets about new features and releases of Requests.
Follow `@kennethreitz `_ for updates.
-Mailing List
-------------
-There's a low-volume mailing list for Requests. To subscribe to the
-mailing list, send an email to
-`requests@librelist.org `_.
-
-
-
-
-Software Updates
-================
+Release and Version History
+===========================
.. include:: ../../HISTORY.rst
diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst
new file mode 100644
index 00000000..53b2a8e5
--- /dev/null
+++ b/docs/dev/contributing.rst
@@ -0,0 +1,145 @@
+.. _contributing:
+
+Contributor's Guide
+===================
+
+If you're reading this you're probably interested in contributing to
+Requests. First, We'd like to say: thank you! Open source projects
+live-and-die based on the support they receive from others, and the fact that
+you're even considering supporting Requests is very generous of
+you.
+
+This document lays out guidelines and advice for contributing to Requests.
+If you're thinking of contributing, start by reading this thoroughly and
+getting a feel for how contributing to the project works. If you have any
+questions, feel free to reach out to either `Ian Cordasco`_ or `Cory Benfield`_,
+the primary maintainers.
+
+The guide is split into sections based on the type of contribution you're
+thinking of making, with a section that covers general guidelines for all
+contributors.
+
+.. _Ian Cordasco: http://www.coglib.com/~icordasc/
+.. _Cory Benfield: https://lukasa.co.uk/about
+
+
+All Contributions
+-----------------
+
+Be Cordial
+~~~~~~~~~~
+
+**Be cordial or be on your way.**
+
+Requests has one very important rule governing all forms of contribution,
+including reporting bugs or requesting features. This golden rule is
+`be cordial or be on your way`_. **All contributions are welcome**, as long as
+everyone involved is treated with respect.
+
+.. _be cordial or be on your way: http://kennethreitz.org/be-cordial-or-be-on-your-way/
+
+.. _early-feedback:
+
+Get Early Feedback
+~~~~~~~~~~~~~~~~~~
+
+If you are contributing, do not feel the need to sit on your contribution until
+it is perfectly polished and complete. It helps everyone involved for you to
+seek feedback as early as you possibly can. Submitting an early, unfinished
+version of your contribution for feedback in no way prejudices your chances of
+getting that contribution accepted, and can save you from putting a lot of work
+into a contribution that is not suitable for the project.
+
+Contribution Suitability
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The project maintainer has the last word on whether or not a contribution is
+suitable for Requests. All contributions will be considered, but from time
+to time contributions will be rejected because they do not suit the project.
+
+If your contribution is rejected, don't despair! So long as you followed these
+guidelines, you'll have a much better chance of getting your next contribution
+accepted.
+
+
+Code Contributions
+------------------
+
+Steps
+~~~~~
+
+When contributing code, you'll want to follow this checklist:
+
+1. Fork the repository on GitHub.
+2. Run the tests to confirm they all pass on your system. If they don't, you'll
+ need to investigate why they fail. If you're unable to diagnose this
+ yourself, raise it as a bug report by following the guidelines in this
+ document: :ref:`bug-reports`.
+3. Write tests that demonstrate your bug or feature. Ensure that they fail.
+4. Make your change.
+5. Run the entire test suite again, confirming that all tests pass *including
+ the ones you just added*.
+6. Send a GitHub Pull Request to the main repository's ``master`` branch.
+ GitHub Pull Requests are the expected method of code collaboration on this
+ project.
+
+The following sub-sections go into more detail on some of the points above.
+
+Code Review
+~~~~~~~~~~~
+
+Contributions will not be merged until they've been code reviewed. You should
+implement any code review feedback unless you strongly object to it. In the
+event that you object to the code review feedback, you should make your case
+clearly and calmly. If, after doing so, the feedback is judged to still apply,
+you must either apply the feedback or withdraw your contribution.
+
+New Contributors
+~~~~~~~~~~~~~~~~
+
+If you are new or relatively new to Open Source, welcome! Requests aims to
+be a gentle introduction to the world of Open Source. If you're concerned about
+how best to contribute, please consider mailing a maintainer (listed above) and
+asking for help.
+
+Please also check the :ref:`early-feedback` section.
+
+Documentation Contributions
+---------------------------
+
+Documentation improvements are always welcome! The documentation files live in
+the ``docs/`` directory of the codebase. They're written in
+`reStructuredText`_, and use `Sphinx`_ to generate the full suite of
+documentation.
+
+When contributing documentation, please attempt to follow the style of the
+documentation files. This means a soft-limit of 79 characters wide in your text
+files and a semi-formal prose style.
+
+.. _reStructuredText: http://docutils.sourceforge.net/rst.html
+.. _Sphinx: http://sphinx-doc.org/index.html
+
+
+.. _bug-reports:
+
+Bug Reports
+-----------
+
+Bug reports are hugely important! Before you raise one, though, please check
+through the `GitHub issues`_, **both open and closed**, to confirm that the bug
+hasn't been reported before. Duplicate bug reports are a huge drain on the time
+of other contributors, and should be avoided as much as possible.
+
+.. _GitHub issues: https://github.com/kennethreitz/requests/issues
+
+
+Feature Requests
+----------------
+
+Requests is in a perpeptual feature freeze. The maintainers believe that
+requests contains every major feature currently required by the vast majority
+of users.
+
+If you believe there is a feature missing, feel free to raise a feature
+request, but please do be aware that the overwhelming likelihood is that your
+feature request will not be accepted.
diff --git a/docs/index.rst b/docs/index.rst
index 12b24c9e..04ba7e50 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -115,10 +115,11 @@ Requests ecosystem and community.
community/faq
community/recommended
- community/out-there.rst
+ community/out-there
community/support
community/vulnerabilities
community/updates
+ community/release-process
API Documentation
-----------------
@@ -141,6 +142,7 @@ you.
.. toctree::
:maxdepth: 1
+ dev/contributing
dev/philosophy
dev/todo
dev/authors
diff --git a/requests/__init__.py b/requests/__init__.py
index a403663b..d2471284 100644
--- a/requests/__init__.py
+++ b/requests/__init__.py
@@ -42,8 +42,8 @@ is at .
"""
__title__ = 'requests'
-__version__ = '2.6.0'
-__build__ = 0x020600
+__version__ = '2.7.0'
+__build__ = 0x020700
__author__ = 'Kenneth Reitz'
__license__ = 'Apache 2.0'
__copyright__ = 'Copyright 2015 Kenneth Reitz'
diff --git a/requests/api.py b/requests/api.py
index 98c92298..d40fa380 100644
--- a/requests/api.py
+++ b/requests/api.py
@@ -55,17 +55,18 @@ def request(method, url, **kwargs):
return response
-def get(url, **kwargs):
+def get(url, params=None, **kwargs):
"""Sends a GET request.
:param url: URL for the new :class:`Request` object.
+ :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
:param \*\*kwargs: Optional arguments that ``request`` takes.
:return: :class:`Response ` object
:rtype: requests.Response
"""
kwargs.setdefault('allow_redirects', True)
- return request('get', url, **kwargs)
+ return request('get', url, params=params, **kwargs)
def options(url, **kwargs):
diff --git a/requests/models.py b/requests/models.py
index cb05e030..f32dd5f6 100644
--- a/requests/models.py
+++ b/requests/models.py
@@ -513,6 +513,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
def prepare_hooks(self, hooks):
"""Prepares the given hooks."""
+ # hooks can be passed as None to the prepare method and to this
+ # method. To prevent iterating over None, simply use an empty list
+ # if hooks is False-y
+ hooks = hooks or []
for event in hooks:
self.register_hook(event, hooks[event])
diff --git a/requests/packages/__init__.py b/requests/packages/__init__.py
index b73b4d1b..d62c4b71 100644
--- a/requests/packages/__init__.py
+++ b/requests/packages/__init__.py
@@ -1,107 +1,3 @@
-"""
-Copyright (c) Donald Stufft, pip, and individual contributors
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-"""
from __future__ import absolute_import
-import sys
-
-
-class VendorAlias(object):
-
- def __init__(self, package_names):
- self._package_names = package_names
- self._vendor_name = __name__
- self._vendor_pkg = self._vendor_name + "."
- self._vendor_pkgs = [
- self._vendor_pkg + name for name in self._package_names
- ]
-
- def find_module(self, fullname, path=None):
- if fullname.startswith(self._vendor_pkg):
- return self
-
- def load_module(self, name):
- # Ensure that this only works for the vendored name
- if not name.startswith(self._vendor_pkg):
- raise ImportError(
- "Cannot import %s, must be a subpackage of '%s'." % (
- name, self._vendor_name,
- )
- )
-
- if not (name == self._vendor_name or
- any(name.startswith(pkg) for pkg in self._vendor_pkgs)):
- raise ImportError(
- "Cannot import %s, must be one of %s." % (
- name, self._vendor_pkgs
- )
- )
-
- # Check to see if we already have this item in sys.modules, if we do
- # then simply return that.
- if name in sys.modules:
- return sys.modules[name]
-
- # Check to see if we can import the vendor name
- try:
- # We do this dance here because we want to try and import this
- # module without hitting a recursion error because of a bunch of
- # VendorAlias instances on sys.meta_path
- real_meta_path = sys.meta_path[:]
- try:
- sys.meta_path = [
- m for m in sys.meta_path
- if not isinstance(m, VendorAlias)
- ]
- __import__(name)
- module = sys.modules[name]
- finally:
- # Re-add any additions to sys.meta_path that were made while
- # during the import we just did, otherwise things like
- # requests.packages.urllib3.poolmanager will fail.
- for m in sys.meta_path:
- if m not in real_meta_path:
- real_meta_path.append(m)
-
- # Restore sys.meta_path with any new items.
- sys.meta_path = real_meta_path
- except ImportError:
- # We can't import the vendor name, so we'll try to import the
- # "real" name.
- real_name = name[len(self._vendor_pkg):]
- try:
- __import__(real_name)
- module = sys.modules[real_name]
- except ImportError:
- raise ImportError("No module named '%s'" % (name,))
-
- # If we've gotten here we've found the module we're looking for, either
- # as part of our vendored package, or as the real name, so we'll add
- # it to sys.modules as the vendored name so that we don't have to do
- # the lookup again.
- sys.modules[name] = module
-
- # Finally, return the loaded module
- return module
-
-
-sys.meta_path.insert(0, VendorAlias(["urllib3", "chardet"]))
+from . import urllib3
diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py
index 0660b9c8..f48ac4af 100644
--- a/requests/packages/urllib3/__init__.py
+++ b/requests/packages/urllib3/__init__.py
@@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using.
__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
__license__ = 'MIT'
-__version__ = '1.10.2'
+__version__ = '1.10.4'
from .connectionpool import (
@@ -55,9 +55,12 @@ def add_stderr_logger(level=logging.DEBUG):
del NullHandler
-# Set security warning to always go off by default.
import warnings
-warnings.simplefilter('always', exceptions.SecurityWarning)
+# SecurityWarning's always go off by default.
+warnings.simplefilter('always', exceptions.SecurityWarning, append=True)
+# InsecurePlatformWarning's don't vary between requests, so we keep it default.
+warnings.simplefilter('default', exceptions.InsecurePlatformWarning,
+ append=True)
def disable_warnings(category=exceptions.HTTPWarning):
"""
diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py
index cc424de0..279416ce 100644
--- a/requests/packages/urllib3/_collections.py
+++ b/requests/packages/urllib3/_collections.py
@@ -227,20 +227,20 @@ class HTTPHeaderDict(dict):
# Need to convert the tuple to list for further extension
_dict_setitem(self, key_lower, [vals[0], vals[1], val])
- def extend(*args, **kwargs):
+ def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
- if len(args) > 2:
- raise TypeError("update() takes at most 2 positional "
+ if len(args) > 1:
+ raise TypeError("extend() takes at most 1 positional "
"arguments ({} given)".format(len(args)))
- elif not args:
- raise TypeError("update() takes at least 1 argument (0 given)")
- self = args[0]
- other = args[1] if len(args) >= 2 else ()
+ other = args[0] if len(args) >= 1 else ()
- if isinstance(other, Mapping):
+ if isinstance(other, HTTPHeaderDict):
+ for key, val in other.iteritems():
+ self.add(key, val)
+ elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
@@ -304,17 +304,20 @@ class HTTPHeaderDict(dict):
return list(self.iteritems())
@classmethod
- def from_httplib(cls, message, duplicates=('set-cookie',)): # Python 2
+ def from_httplib(cls, message): # Python 2
"""Read headers from a Python 2 httplib message object."""
- ret = cls(message.items())
- # ret now contains only the last header line for each duplicate.
- # Importing with all duplicates would be nice, but this would
- # mean to repeat most of the raw parsing already done, when the
- # message object was created. Extracting only the headers of interest
- # separately, the cookies, should be faster and requires less
- # extra code.
- for key in duplicates:
- ret.discard(key)
- for val in message.getheaders(key):
- ret.add(key, val)
- return ret
+ # python2.7 does not expose a proper API for exporting multiheaders
+ # efficiently. This function re-reads raw lines from the message
+ # object and extracts the multiheaders properly.
+ headers = []
+
+ for line in message.headers:
+ if line.startswith((' ', '\t')):
+ key, value = headers[-1]
+ headers[-1] = (key, value + '\r\n' + line.rstrip())
+ continue
+
+ key, value = line.split(':', 1)
+ headers.append((key, value.strip()))
+
+ return cls(headers)
diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py
index e5de769d..2a8c3596 100644
--- a/requests/packages/urllib3/connection.py
+++ b/requests/packages/urllib3/connection.py
@@ -260,3 +260,5 @@ if ssl:
# Make a copy for testing.
UnverifiedHTTPSConnection = HTTPSConnection
HTTPSConnection = VerifiedHTTPSConnection
+else:
+ HTTPSConnection = DummyConnection
diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py
index 0085345c..117269ac 100644
--- a/requests/packages/urllib3/connectionpool.py
+++ b/requests/packages/urllib3/connectionpool.py
@@ -735,7 +735,6 @@ class HTTPSConnectionPool(HTTPConnectionPool):
% (self.num_connections, self.host))
if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
- # Platform-specific: Python without ssl
raise SSLError("Can't connect to HTTPS URL because the SSL "
"module is not available.")
diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py
index ee657fb3..b2c34a89 100644
--- a/requests/packages/urllib3/contrib/pyopenssl.py
+++ b/requests/packages/urllib3/contrib/pyopenssl.py
@@ -38,8 +38,6 @@ Module Variables
----------------
:var DEFAULT_SSL_CIPHER_LIST: The list of supported SSL/TLS cipher suites.
- Default: ``ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:
- ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS``
.. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication
.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit)
@@ -85,22 +83,7 @@ _openssl_verify = {
+ OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
}
-# A secure default.
-# Sources for more information on TLS ciphers:
-#
-# - https://wiki.mozilla.org/Security/Server_Side_TLS
-# - https://www.ssllabs.com/projects/best-practices/index.html
-# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
-#
-# The general intent is:
-# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE),
-# - prefer ECDHE over DHE for better performance,
-# - prefer any AES-GCM over any AES-CBC for better performance and security,
-# - use 3DES as fallback which is secure but slow,
-# - disable NULL authentication, MD5 MACs and DSS for security reasons.
-DEFAULT_SSL_CIPHER_LIST = "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:" + \
- "ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:" + \
- "!aNULL:!MD5:!DSS"
+DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS
orig_util_HAS_SNI = util.HAS_SNI
@@ -299,7 +282,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
try:
cnx.do_handshake()
except OpenSSL.SSL.WantReadError:
- select.select([sock], [], [])
+ rd, _, _ = select.select([sock], [], [], sock.gettimeout())
+ if not rd:
+ raise timeout('select timed out')
continue
except OpenSSL.SSL.Error as e:
raise ssl.SSLError('bad handshake', e)
diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py
index 5d523011..31bda1c0 100644
--- a/requests/packages/urllib3/exceptions.py
+++ b/requests/packages/urllib3/exceptions.py
@@ -162,3 +162,8 @@ class SystemTimeWarning(SecurityWarning):
class InsecurePlatformWarning(SecurityWarning):
"Warned when certain SSL configuration is not available on a platform."
pass
+
+
+class ResponseNotChunked(ProtocolError, ValueError):
+ "Response needs to be chunked in order to read it as chunks."
+ pass
diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py
index 34cd3d70..24140c4c 100644
--- a/requests/packages/urllib3/response.py
+++ b/requests/packages/urllib3/response.py
@@ -1,9 +1,15 @@
+try:
+ import http.client as httplib
+except ImportError:
+ import httplib
import zlib
import io
from socket import timeout as SocketTimeout
from ._collections import HTTPHeaderDict
-from .exceptions import ProtocolError, DecodeError, ReadTimeoutError
+from .exceptions import (
+ ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked
+)
from .packages.six import string_types as basestring, binary_type, PY3
from .connection import HTTPException, BaseSSLError
from .util.response import is_fp_closed
@@ -117,7 +123,17 @@ class HTTPResponse(io.IOBase):
if hasattr(body, 'read'):
self._fp = body
- if preload_content and not self._body:
+ # Are we using the chunked-style of transfer encoding?
+ self.chunked = False
+ self.chunk_left = None
+ tr_enc = self.headers.get('transfer-encoding', '').lower()
+ # Don't incur the penalty of creating a list and then discarding it
+ encodings = (enc.strip() for enc in tr_enc.split(","))
+ if "chunked" in encodings:
+ self.chunked = True
+
+ # We certainly don't want to preload content when the response is chunked.
+ if not self.chunked and preload_content and not self._body:
self._body = self.read(decode_content=decode_content)
def get_redirect_location(self):
@@ -157,6 +173,35 @@ class HTTPResponse(io.IOBase):
"""
return self._fp_bytes_read
+ def _init_decoder(self):
+ """
+ Set-up the _decoder attribute if necessar.
+ """
+ # Note: content-encoding value should be case-insensitive, per RFC 7230
+ # Section 3.2
+ content_encoding = self.headers.get('content-encoding', '').lower()
+ if self._decoder is None and content_encoding in self.CONTENT_DECODERS:
+ self._decoder = _get_decoder(content_encoding)
+
+ def _decode(self, data, decode_content, flush_decoder):
+ """
+ Decode the data passed in and potentially flush the decoder.
+ """
+ try:
+ if decode_content and self._decoder:
+ data = self._decoder.decompress(data)
+ except (IOError, zlib.error) as e:
+ content_encoding = self.headers.get('content-encoding', '').lower()
+ raise DecodeError(
+ "Received response with content-encoding: %s, but "
+ "failed to decode it." % content_encoding, e)
+
+ if flush_decoder and decode_content and self._decoder:
+ buf = self._decoder.decompress(binary_type())
+ data += buf + self._decoder.flush()
+
+ return data
+
def read(self, amt=None, decode_content=None, cache_content=False):
"""
Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
@@ -178,12 +223,7 @@ class HTTPResponse(io.IOBase):
after having ``.read()`` the file object. (Overridden if ``amt`` is
set.)
"""
- # Note: content-encoding value should be case-insensitive, per RFC 7230
- # Section 3.2
- content_encoding = self.headers.get('content-encoding', '').lower()
- if self._decoder is None:
- if content_encoding in self.CONTENT_DECODERS:
- self._decoder = _get_decoder(content_encoding)
+ self._init_decoder()
if decode_content is None:
decode_content = self.decode_content
@@ -232,17 +272,7 @@ class HTTPResponse(io.IOBase):
self._fp_bytes_read += len(data)
- try:
- if decode_content and self._decoder:
- data = self._decoder.decompress(data)
- except (IOError, zlib.error) as e:
- raise DecodeError(
- "Received response with content-encoding: %s, but "
- "failed to decode it." % content_encoding, e)
-
- if flush_decoder and decode_content and self._decoder:
- buf = self._decoder.decompress(binary_type())
- data += buf + self._decoder.flush()
+ data = self._decode(data, decode_content, flush_decoder)
if cache_content:
self._body = data
@@ -269,11 +299,15 @@ class HTTPResponse(io.IOBase):
If True, will attempt to decode the body based on the
'content-encoding' header.
"""
- while not is_fp_closed(self._fp):
- data = self.read(amt=amt, decode_content=decode_content)
+ if self.chunked:
+ for line in self.read_chunked(amt, decode_content=decode_content):
+ yield line
+ else:
+ while not is_fp_closed(self._fp):
+ data = self.read(amt=amt, decode_content=decode_content)
- if data:
- yield data
+ if data:
+ yield data
@classmethod
def from_httplib(ResponseCls, r, **response_kw):
@@ -351,3 +385,82 @@ class HTTPResponse(io.IOBase):
else:
b[:len(temp)] = temp
return len(temp)
+
+ def _update_chunk_length(self):
+ # First, we'll figure out length of a chunk and then
+ # we'll try to read it from socket.
+ if self.chunk_left is not None:
+ return
+ line = self._fp.fp.readline()
+ line = line.split(b';', 1)[0]
+ try:
+ self.chunk_left = int(line, 16)
+ except ValueError:
+ # Invalid chunked protocol response, abort.
+ self.close()
+ raise httplib.IncompleteRead(line)
+
+ def _handle_chunk(self, amt):
+ returned_chunk = None
+ if amt is None:
+ chunk = self._fp._safe_read(self.chunk_left)
+ returned_chunk = chunk
+ self._fp._safe_read(2) # Toss the CRLF at the end of the chunk.
+ self.chunk_left = None
+ elif amt < self.chunk_left:
+ value = self._fp._safe_read(amt)
+ self.chunk_left = self.chunk_left - amt
+ returned_chunk = value
+ elif amt == self.chunk_left:
+ value = self._fp._safe_read(amt)
+ self._fp._safe_read(2) # Toss the CRLF at the end of the chunk.
+ self.chunk_left = None
+ returned_chunk = value
+ else: # amt > self.chunk_left
+ returned_chunk = self._fp._safe_read(self.chunk_left)
+ self._fp._safe_read(2) # Toss the CRLF at the end of the chunk.
+ self.chunk_left = None
+ return returned_chunk
+
+ def read_chunked(self, amt=None, decode_content=None):
+ """
+ Similar to :meth:`HTTPResponse.read`, but with an additional
+ parameter: ``decode_content``.
+
+ :param decode_content:
+ If True, will attempt to decode the body based on the
+ 'content-encoding' header.
+ """
+ self._init_decoder()
+ # FIXME: Rewrite this method and make it a class with a better structured logic.
+ if not self.chunked:
+ raise ResponseNotChunked("Response is not chunked. "
+ "Header 'transfer-encoding: chunked' is missing.")
+
+ if self._original_response and self._original_response._method.upper() == 'HEAD':
+ # Don't bother reading the body of a HEAD request.
+ # FIXME: Can we do this somehow without accessing private httplib _method?
+ self._original_response.close()
+ return
+
+ while True:
+ self._update_chunk_length()
+ if self.chunk_left == 0:
+ break
+ chunk = self._handle_chunk(amt)
+ yield self._decode(chunk, decode_content=decode_content,
+ flush_decoder=True)
+
+ # Chunk content ends with \r\n: discard it.
+ while True:
+ line = self._fp.fp.readline()
+ if not line:
+ # Some sites may not end with '\r\n'.
+ break
+ if line == b'\r\n':
+ break
+
+ # We read everything; close the "file".
+ if self._original_response:
+ self._original_response.close()
+ self.release_conn()
diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py
index e7e7dfae..b846d42c 100644
--- a/requests/packages/urllib3/util/ssl_.py
+++ b/requests/packages/urllib3/util/ssl_.py
@@ -9,10 +9,10 @@ HAS_SNI = False
create_default_context = None
import errno
-import ssl
import warnings
try: # Test for SSL features
+ import ssl
from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
from ssl import HAS_SNI # Has SNI?
except ImportError:
@@ -25,14 +25,24 @@ except ImportError:
OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
OP_NO_COMPRESSION = 0x20000
-try:
- from ssl import _DEFAULT_CIPHERS
-except ImportError:
- _DEFAULT_CIPHERS = (
- 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
- 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:'
- '!eNULL:!MD5'
- )
+# A secure default.
+# Sources for more information on TLS ciphers:
+#
+# - https://wiki.mozilla.org/Security/Server_Side_TLS
+# - https://www.ssllabs.com/projects/best-practices/index.html
+# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
+#
+# The general intent is:
+# - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE),
+# - prefer ECDHE over DHE for better performance,
+# - prefer any AES-GCM over any AES-CBC for better performance and security,
+# - use 3DES as fallback which is secure but slow,
+# - disable NULL authentication, MD5 MACs and DSS for security reasons.
+DEFAULT_CIPHERS = (
+ 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
+ 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:'
+ '!eNULL:!MD5'
+)
try:
from ssl import SSLContext # Modern SSL?
@@ -40,7 +50,8 @@ except ImportError:
import sys
class SSLContext(object): # Platform-specific: Python 2 & 3.1
- supports_set_ciphers = sys.version_info >= (2, 7)
+ supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or
+ (3, 2) <= sys.version_info)
def __init__(self, protocol_version):
self.protocol = protocol_version
@@ -167,7 +178,7 @@ def resolve_ssl_version(candidate):
return candidate
-def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED,
+def create_urllib3_context(ssl_version=None, cert_reqs=None,
options=None, ciphers=None):
"""All arguments have the same meaning as ``ssl_wrap_socket``.
@@ -204,6 +215,9 @@ def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED,
"""
context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23)
+ # Setting the default here, as we may have no ssl module on import
+ cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs
+
if options is None:
options = 0
# SSLv2 is easily broken and is considered harmful and dangerous
@@ -217,7 +231,7 @@ def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED,
context.options |= options
if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6
- context.set_ciphers(ciphers or _DEFAULT_CIPHERS)
+ context.set_ciphers(ciphers or DEFAULT_CIPHERS)
context.verify_mode = cert_reqs
if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2
diff --git a/requests/packages/urllib3/util/url.py b/requests/packages/urllib3/util/url.py
index b2ec834f..e58050cd 100644
--- a/requests/packages/urllib3/util/url.py
+++ b/requests/packages/urllib3/util/url.py
@@ -15,6 +15,8 @@ class Url(namedtuple('Url', url_attrs)):
def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None,
query=None, fragment=None):
+ if path and not path.startswith('/'):
+ path = '/' + path
return super(Url, cls).__new__(cls, scheme, auth, host, port, path,
query, fragment)
diff --git a/test_requests.py b/test_requests.py
index 15406a22..cad8c055 100755
--- a/test_requests.py
+++ b/test_requests.py
@@ -1613,7 +1613,6 @@ def test_prepare_unicode_url():
p.prepare(
method='GET',
url=u('http://www.example.com/üniçø∂é'),
- hooks=[]
)
assert_copy(p, p.copy())