Update urllib3 to 1.19

requests/packages/urllib3/__init__.py

@@ -32,7 +32,7 @@ except ImportError:
 
 __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
 __license__ = 'MIT'
-__version__ = '1.16'
+__version__ = '1.19'
 
 __all__ = (
     'HTTPConnectionPool',

requests/packages/urllib3/connection.py

@@ -7,13 +7,8 @@ import socket
 from socket import error as SocketError, timeout as SocketTimeout
 import warnings
 from .packages import six
-
-try:  # Python 3
-    from http.client import HTTPConnection as _HTTPConnection
-    from http.client import HTTPException  # noqa: unused in this module
-except ImportError:
-    from httplib import HTTPConnection as _HTTPConnection
-    from httplib import HTTPException  # noqa: unused in this module
+from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection
+from .packages.six.moves.http_client import HTTPException  # noqa: F401
 
 try:  # Compiled with SSL?
     import ssl
@@ -44,8 +39,9 @@ from .packages.ssl_match_hostname import match_hostname, CertificateError
 from .util.ssl_ import (
     resolve_cert_reqs,
     resolve_ssl_version,
-    ssl_wrap_socket,
     assert_fingerprint,
+    create_urllib3_context,
+    ssl_wrap_socket
 )
 
 
@@ -174,7 +170,13 @@ class HTTPConnection(_HTTPConnection, object):
         """
         headers = HTTPHeaderDict(headers if headers is not None else {})
         skip_accept_encoding = 'accept-encoding' in headers
-        self.putrequest(method, url, skip_accept_encoding=skip_accept_encoding)
+        skip_host = 'host' in headers
+        self.putrequest(
+            method,
+            url,
+            skip_accept_encoding=skip_accept_encoding,
+            skip_host=skip_host
+        )
         for header, value in headers.items():
             self.putheader(header, value)
         if 'transfer-encoding' not in headers:
@@ -203,14 +205,18 @@ class HTTPConnection(_HTTPConnection, object):
 class HTTPSConnection(HTTPConnection):
     default_port = port_by_scheme['https']
 
+    ssl_version = None
+
     def __init__(self, host, port=None, key_file=None, cert_file=None,
-                 strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, **kw):
+                 strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                 ssl_context=None, **kw):
 
         HTTPConnection.__init__(self, host, port, strict=strict,
                                 timeout=timeout, **kw)
 
         self.key_file = key_file
         self.cert_file = cert_file
+        self.ssl_context = ssl_context
 
         # Required property for Google AppEngine 1.9.0 which otherwise causes
         # HTTPS requests to go out as HTTP. (See Issue #356)
@@ -219,7 +225,19 @@ class HTTPSConnection(HTTPConnection):
     def connect(self):
         conn = self._new_conn()
         self._prepare_conn(conn)
-        self.sock = ssl.wrap_socket(conn, self.key_file, self.cert_file)
+
+        if self.ssl_context is None:
+            self.ssl_context = create_urllib3_context(
+                ssl_version=resolve_ssl_version(None),
+                cert_reqs=resolve_cert_reqs(None),
+            )
+
+        self.sock = ssl_wrap_socket(
+            sock=conn,
+            keyfile=self.key_file,
+            certfile=self.cert_file,
+            ssl_context=self.ssl_context,
+        )
 
 
 class VerifiedHTTPSConnection(HTTPSConnection):
@@ -237,9 +255,18 @@ class VerifiedHTTPSConnection(HTTPSConnection):
                  cert_reqs=None, ca_certs=None,
                  assert_hostname=None, assert_fingerprint=None,
                  ca_cert_dir=None):
-
-        if (ca_certs or ca_cert_dir) and cert_reqs is None:
-            cert_reqs = 'CERT_REQUIRED'
+        """
+        This method should only be called once, before the connection is used.
+        """
+        # If cert_reqs is not provided, we can try to guess. If the user gave
+        # us a cert database, we assume they want to use it: otherwise, if
+        # they gave us an SSL Context object we should use whatever is set for
+        # it.
+        if cert_reqs is None:
+            if ca_certs or ca_cert_dir:
+                cert_reqs = 'CERT_REQUIRED'
+            elif self.ssl_context is not None:
+                cert_reqs = self.ssl_context.verify_mode
 
         self.key_file = key_file
         self.cert_file = cert_file
@@ -253,9 +280,6 @@ class VerifiedHTTPSConnection(HTTPSConnection):
         # Add certificate verification
         conn = self._new_conn()
 
-        resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs)
-        resolved_ssl_version = resolve_ssl_version(self.ssl_version)
-
         hostname = self.host
         if getattr(self, '_tunnel_host', None):
             # _tunnel_host was added in Python 2.6.3
@@ -281,17 +305,27 @@ class VerifiedHTTPSConnection(HTTPSConnection):
 
         # Wrap socket using verification with the root certs in
         # trusted_root_certs
-        self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file,
-                                    cert_reqs=resolved_cert_reqs,
-                                    ca_certs=self.ca_certs,
-                                    ca_cert_dir=self.ca_cert_dir,
-                                    server_hostname=hostname,
-                                    ssl_version=resolved_ssl_version)
+        if self.ssl_context is None:
+            self.ssl_context = create_urllib3_context(
+                ssl_version=resolve_ssl_version(self.ssl_version),
+                cert_reqs=resolve_cert_reqs(self.cert_reqs),
+            )
+
+        context = self.ssl_context
+        context.verify_mode = resolve_cert_reqs(self.cert_reqs)
+        self.sock = ssl_wrap_socket(
+            sock=conn,
+            keyfile=self.key_file,
+            certfile=self.cert_file,
+            ca_certs=self.ca_certs,
+            ca_cert_dir=self.ca_cert_dir,
+            server_hostname=hostname,
+            ssl_context=context)
 
         if self.assert_fingerprint:
             assert_fingerprint(self.sock.getpeercert(binary_form=True),
                                self.assert_fingerprint)
-        elif resolved_cert_reqs != ssl.CERT_NONE \
+        elif context.verify_mode != ssl.CERT_NONE \
                 and self.assert_hostname is not False:
             cert = self.sock.getpeercert()
             if not cert.get('subjectAltName', ()):
@@ -304,8 +338,10 @@ class VerifiedHTTPSConnection(HTTPSConnection):
                 )
             _match_hostname(cert, self.assert_hostname or hostname)
 
-        self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or
-                            self.assert_fingerprint is not None)
+        self.is_verified = (
+            context.verify_mode == ssl.CERT_REQUIRED or
+            self.assert_fingerprint is not None
+        )
 
 
 def _match_hostname(cert, asserted_hostname):

requests/packages/urllib3/connectionpool.py

@@ -7,13 +7,6 @@ import warnings
 from socket import error as SocketError, timeout as SocketTimeout
 import socket
 
-try:  # Python 3
-    from queue import LifoQueue, Empty, Full
-except ImportError:
-    from Queue import LifoQueue, Empty, Full
-    # Queue is imported for side effects on MS Windows
-    import Queue as _unused_module_Queue  # noqa: unused
-
 
 from .exceptions import (
     ClosedPoolError,
@@ -32,6 +25,7 @@ from .exceptions import (
 )
 from .packages.ssl_match_hostname import CertificateError
 from .packages import six
+from .packages.six.moves.queue import LifoQueue, Empty, Full
 from .connection import (
     port_by_scheme,
     DummyConnection,
@@ -48,6 +42,10 @@ from .util.timeout import Timeout
 from .util.url import get_host, Url
 
 
+if six.PY2:
+    # Queue is imported for side effects on MS Windows
+    import Queue as _unused_module_Queue  # noqa: F401
+
 xrange = six.moves.xrange
 
 log = logging.getLogger(__name__)
@@ -210,8 +208,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         Return a fresh :class:`HTTPConnection`.
         """
         self.num_connections += 1
-        log.info("Starting new HTTP connection (%d): %s",
-                 self.num_connections, self.host)
+        log.debug("Starting new HTTP connection (%d): %s",
+                  self.num_connections, self.host)
 
         conn = self.ConnectionCls(host=self.host, port=self.port,
                                   timeout=self.timeout.connect_timeout,
@@ -246,7 +244,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
 
         # If this is a persistent connection, check if it got disconnected
         if conn and is_connection_dropped(conn):
-            log.info("Resetting dropped connection: %s", self.host)
+            log.debug("Resetting dropped connection: %s", self.host)
             conn.close()
             if getattr(conn, 'auto_open', 1) == 0:
                 # This is a proxied connection that has been mutated by
@@ -397,8 +395,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
 
         # AppEngine doesn't have a version attr.
         http_version = getattr(conn, '_http_vsn_str', 'HTTP/?')
-        log.debug("\"%s %s %s\" %s %s", method, url, http_version,
-                  httplib_response.status, httplib_response.length)
+        log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port,
+                  method, url, http_version, httplib_response.status,
+                  httplib_response.length)
 
         try:
             assert_header_parsing(httplib_response.msg)
@@ -600,10 +599,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # mess.
             response_conn = conn if not release_conn else None
 
+            # Pass method to Response for length checking
+            response_kw['request_method'] = method
+
             # Import httplib's response into our own wrapper object
             response = self.ResponseCls.from_httplib(httplib_response,
                                                      pool=self,
                                                      connection=response_conn,
+                                                     retries=retries,
                                                      **response_kw)
 
             # Everything went great!
@@ -683,7 +686,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                     raise
                 return response
 
-            log.info("Redirecting %s -> %s", url, redirect_location)
+            retries.sleep_for_retry(response)
+            log.debug("Redirecting %s -> %s", url, redirect_location)
             return self.urlopen(
                 method, redirect_location, body, headers,
                 retries=retries, redirect=redirect,
@@ -692,7 +696,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                 release_conn=release_conn, **response_kw)
 
         # Check if we should retry the HTTP response.
-        if retries.is_forced_retry(method, status_code=response.status):
+        has_retry_after = bool(response.getheader('Retry-After'))
+        if retries.is_retry(method, response.status, has_retry_after):
             try:
                 retries = retries.increment(method, url, response=response, _pool=self)
             except MaxRetryError:
@@ -702,8 +707,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                     response.release_conn()
                     raise
                 return response
-            retries.sleep()
-            log.info("Forced retry: %s", url)
+            retries.sleep(response)
+            log.debug("Retry: %s", url)
             return self.urlopen(
                 method, url, body, headers,
                 retries=retries, redirect=redirect,
@@ -775,7 +780,6 @@ class HTTPSConnectionPool(HTTPConnectionPool):
                           assert_hostname=self.assert_hostname,
                           assert_fingerprint=self.assert_fingerprint)
             conn.ssl_version = self.ssl_version
-
         return conn
 
     def _prepare_proxy(self, conn):
@@ -801,8 +805,8 @@ class HTTPSConnectionPool(HTTPConnectionPool):
         Return a fresh :class:`httplib.HTTPSConnection`.
         """
         self.num_connections += 1
-        log.info("Starting new HTTPS connection (%d): %s",
-                 self.num_connections, self.host)
+        log.debug("Starting new HTTPS connection (%d): %s",
+                  self.num_connections, self.host)
 
         if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
             raise SSLError("Can't connect to HTTPS URL because the SSL "
@@ -834,7 +838,8 @@ class HTTPSConnectionPool(HTTPConnectionPool):
             warnings.warn((
                 'Unverified HTTPS request is being made. '
                 'Adding certificate verification is strongly advised. See: '
-                'https://urllib3.readthedocs.io/en/latest/security.html'),
+                'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
+                '#ssl-warnings'),
                 InsecureRequestWarning)
 
 

requests/packages/urllib3/contrib/appengine.py

@@ -1,7 +1,48 @@
+"""
+This module provides a pool manager that uses Google App Engine's
+`URLFetch Service <https://cloud.google.com/appengine/docs/python/urlfetch>`_.
+
+Example usage::
+
+    from urllib3 import PoolManager
+    from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox
+
+    if is_appengine_sandbox():
+        # AppEngineManager uses AppEngine's URLFetch API behind the scenes
+        http = AppEngineManager()
+    else:
+        # PoolManager uses a socket-level API behind the scenes
+        http = PoolManager()
+
+    r = http.request('GET', 'https://google.com/')
+
+There are `limitations <https://cloud.google.com/appengine/docs/python/\
+urlfetch/#Python_Quotas_and_limits>`_ to the URLFetch service and it may not be
+the best choice for your application. There are three options for using
+urllib3 on Google App Engine:
+
+1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is
+   cost-effective in many circumstances as long as your usage is within the
+   limitations.
+2. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets.
+   Sockets also have `limitations and restrictions
+   <https://cloud.google.com/appengine/docs/python/sockets/\
+   #limitations-and-restrictions>`_ and have a lower free quota than URLFetch.
+   To use sockets, be sure to specify the following in your ``app.yaml``::
+
+        env_variables:
+            GAE_USE_SOCKETS_HTTPLIB : 'true'
+
+3. If you are using `App Engine Flexible
+<https://cloud.google.com/appengine/docs/flexible/>`_, you can use the standard
+:class:`PoolManager` without any configuration or special environment variables.
+"""
+
 from __future__ import absolute_import
 import logging
 import os
 import warnings
+from urlparse import urljoin
 
 from ..exceptions import (
     HTTPError,
@@ -41,13 +82,12 @@ class AppEngineManager(RequestMethods):
 
     This manager uses the URLFetch service directly instead of using the
     emulated httplib, and is subject to URLFetch limitations as described in
-    the App Engine documentation here:
+    the App Engine documentation `here
+    <https://cloud.google.com/appengine/docs/python/urlfetch>`_.
 
-        https://cloud.google.com/appengine/docs/python/urlfetch
-
-    Notably it will raise an AppEnginePlatformError if:
+    Notably it will raise an :class:`AppEnginePlatformError` if:
         * URLFetch is not available.
-        * If you attempt to use this on GAEv2 (Managed VMs), as full socket
+        * If you attempt to use this on App Engine Flexible, as full socket
           support is available.
         * If a request size is more than 10 megabytes.
         * If a response size is more than 32 megabtyes.
@@ -56,7 +96,8 @@ class AppEngineManager(RequestMethods):
     Beyond those cases, it will raise normal urllib3 errors.
     """
 
-    def __init__(self, headers=None, retries=None, validate_certificate=True):
+    def __init__(self, headers=None, retries=None, validate_certificate=True,
+                 urlfetch_retries=True):
         if not urlfetch:
             raise AppEnginePlatformError(
                 "URLFetch is not available in this environment.")
@@ -75,6 +116,7 @@ class AppEngineManager(RequestMethods):
 
         RequestMethods.__init__(self, headers)
         self.validate_certificate = validate_certificate
+        self.urlfetch_retries = urlfetch_retries
 
         self.retries = retries or Retry.DEFAULT
 
@@ -92,16 +134,17 @@ class AppEngineManager(RequestMethods):
         retries = self._get_retries(retries, redirect)
 
         try:
+            follow_redirects = (
+                    redirect and
+                    retries.redirect != 0 and
+                    retries.total)
             response = urlfetch.fetch(
                 url,
                 payload=body,
                 method=method,
                 headers=headers or {},
                 allow_truncated=False,
-                follow_redirects=(
-                    redirect and
-                    retries.redirect != 0 and
-                    retries.total),
+                follow_redirects=self.urlfetch_retries and follow_redirects,
                 deadline=self._get_absolute_timeout(timeout),
                 validate_certificate=self.validate_certificate,
             )
@@ -133,19 +176,40 @@ class AppEngineManager(RequestMethods):
                 "URLFetch does not support method: %s" % method, e)
 
         http_response = self._urlfetch_response_to_http_response(
-            response, **response_kw)
+            response, retries=retries, **response_kw)
 
-        # Check for redirect response
-        if (http_response.get_redirect_location() and
-                retries.raise_on_redirect and redirect):
-            raise MaxRetryError(self, url, "too many redirects")
+        # Handle redirect?
+        redirect_location = redirect and http_response.get_redirect_location()
+        if redirect_location:
+            # Check for redirect response
+            if (self.urlfetch_retries and retries.raise_on_redirect):
+                raise MaxRetryError(self, url, "too many redirects")
+            else:
+                if http_response.status == 303:
+                    method = 'GET'
+
+                try:
+                    retries = retries.increment(method, url, response=http_response, _pool=self)
+                except MaxRetryError:
+                    if retries.raise_on_redirect:
+                        raise MaxRetryError(self, url, "too many redirects")
+                    return http_response
+
+                retries.sleep_for_retry(http_response)
+                log.debug("Redirecting %s -> %s", url, redirect_location)
+                redirect_url = urljoin(url, redirect_location)
+                return self.urlopen(
+                    method, redirect_url, body, headers,
+                    retries=retries, redirect=redirect,
+                    timeout=timeout, **response_kw)
 
         # Check if we should retry the HTTP response.
-        if retries.is_forced_retry(method, status_code=http_response.status):
+        has_retry_after = bool(http_response.getheader('Retry-After'))
+        if retries.is_retry(method, http_response.status, has_retry_after):
             retries = retries.increment(
                 method, url, response=http_response, _pool=self)
-            log.info("Forced retry: %s", url)
-            retries.sleep()
+            log.debug("Retry: %s", url)
+            retries.sleep(http_response)
             return self.urlopen(
                 method, url,
                 body=body, headers=headers,
@@ -183,12 +247,13 @@ class AppEngineManager(RequestMethods):
 
     def _get_absolute_timeout(self, timeout):
         if timeout is Timeout.DEFAULT_TIMEOUT:
-            return 5  # 5s is the default timeout for URLFetch.
+            return None  # Defer to URLFetch's default.
         if isinstance(timeout, Timeout):
-            if timeout._read is not timeout._connect:
+            if timeout._read is not None or timeout._connect is not None:
                 warnings.warn(
                     "URLFetch does not support granular timeout settings, "
-                    "reverting to total timeout.", AppEnginePlatformWarning)
+                    "reverting to total or default URLFetch timeout.",
+                    AppEnginePlatformWarning)
             return timeout.total
         return timeout
 

requests/packages/urllib3/contrib/ntlmpool.py

@@ -5,14 +5,11 @@ Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10
 """
 from __future__ import absolute_import
 
-try:
-    from http.client import HTTPSConnection
-except ImportError:
-    from httplib import HTTPSConnection
 from logging import getLogger
 from ntlm import ntlm
 
-from urllib3 import HTTPSConnectionPool
+from .. import HTTPSConnectionPool
+from ..packages.six.moves.http_client import HTTPSConnection
 
 
 log = getLogger(__name__)

requests/packages/urllib3/contrib/pyopenssl.py

@@ -1,17 +1,21 @@
-'''SSL with SNI_-support for Python 2. Follow these instructions if you would
+"""
+SSL with SNI_-support for Python 2. Follow these instructions if you would
 like to verify SSL certificates in Python 2. Note, the default libraries do
 *not* do certificate checking; you need to do additional work to validate
 certificates yourself.
 
 This needs the following packages installed:
 
-* pyOpenSSL (tested with 0.13)
-* ndg-httpsclient (tested with 0.3.2)
-* pyasn1 (tested with 0.1.6)
+* pyOpenSSL (tested with 16.0.0)
+* cryptography (minimum 1.3.4, from pyopenssl)
+* idna (minimum 2.0, from cryptography)
+
+However, pyopenssl depends on cryptography, which depends on idna, so while we
+use all three directly here we end up having relatively few packages required.
 
 You can install them with the following command:
 
-    pip install pyopenssl ndg-httpsclient pyasn1
+    pip install pyopenssl cryptography idna
 
 To activate certificate checking, call
 :func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code
@@ -34,45 +38,38 @@ compression in Python 2 (see `CRIME attack`_).
 If you want to configure the default list of supported cipher suites, you can
 set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.
 
-Module Variables
-----------------
-
-:var DEFAULT_SSL_CIPHER_LIST: The list of supported SSL/TLS cipher suites.
-
 .. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication
 .. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit)
-
-'''
+"""
 from __future__ import absolute_import
 
-try:
-    from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
-    from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName
-except SyntaxError as e:
-    raise ImportError(e)
-
+import idna
 import OpenSSL.SSL
-from pyasn1.codec.der import decoder as der_decoder
-from pyasn1.type import univ, constraint
+from cryptography import x509
+from cryptography.hazmat.backends.openssl import backend as openssl_backend
+from cryptography.hazmat.backends.openssl.x509 import _Certificate
+
 from socket import timeout, error as SocketError
+from io import BytesIO
 
 try:  # Platform-specific: Python 2
     from socket import _fileobject
 except ImportError:  # Platform-specific: Python 3
     _fileobject = None
-    from urllib3.packages.backports.makefile import backport_makefile
+    from ..packages.backports.makefile import backport_makefile
 
+import logging
 import ssl
 import select
 import six
+import sys
 
-from .. import connection
 from .. import util
 
 __all__ = ['inject_into_urllib3', 'extract_from_urllib3']
 
-# SNI only *really* works if we can read the subjectAltName of certificates.
-HAS_SNI = SUBJ_ALT_NAME_SUPPORT
+# SNI always works.
+HAS_SNI = True
 
 # Map from urllib3 to PyOpenSSL compatible parameter-values.
 _openssl_versions = {
@@ -91,78 +88,120 @@ try:
 except AttributeError:
     pass
 
-_openssl_verify = {
+_stdlib_to_openssl_verify = {
     ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
     ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
     ssl.CERT_REQUIRED:
         OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
 }
-
-DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS.encode('ascii')
+_openssl_to_stdlib_verify = dict(
+    (v, k) for k, v in _stdlib_to_openssl_verify.items()
+)
 
 # OpenSSL will only write 16K at a time
 SSL_WRITE_BLOCKSIZE = 16384
 
 orig_util_HAS_SNI = util.HAS_SNI
-orig_connection_ssl_wrap_socket = connection.ssl_wrap_socket
+orig_util_SSLContext = util.ssl_.SSLContext
+
+
+log = logging.getLogger(__name__)
 
 
 def inject_into_urllib3():
     'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.'
 
-    connection.ssl_wrap_socket = ssl_wrap_socket
+    util.ssl_.SSLContext = PyOpenSSLContext
     util.HAS_SNI = HAS_SNI
+    util.ssl_.HAS_SNI = HAS_SNI
     util.IS_PYOPENSSL = True
+    util.ssl_.IS_PYOPENSSL = True
 
 
 def extract_from_urllib3():
     'Undo monkey-patching by :func:`inject_into_urllib3`.'
 
-    connection.ssl_wrap_socket = orig_connection_ssl_wrap_socket
+    util.ssl_.SSLContext = orig_util_SSLContext
     util.HAS_SNI = orig_util_HAS_SNI
+    util.ssl_.HAS_SNI = orig_util_HAS_SNI
     util.IS_PYOPENSSL = False
+    util.ssl_.IS_PYOPENSSL = False
 
 
-# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
-class SubjectAltName(BaseSubjectAltName):
-    '''ASN.1 implementation for subjectAltNames support'''
+def _dnsname_to_stdlib(name):
+    """
+    Converts a dNSName SubjectAlternativeName field to the form used by the
+    standard library on the given Python version.
 
-    # There is no limit to how many SAN certificates a certificate may have,
-    #   however this needs to have some limit so we'll set an arbitrarily high
-    #   limit.
-    sizeSpec = univ.SequenceOf.sizeSpec + \
-        constraint.ValueSizeConstraint(1, 1024)
+    Cryptography produces a dNSName as a unicode string that was idna-decoded
+    from ASCII bytes. We need to idna-encode that string to get it back, and
+    then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib
+    uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8).
+    """
+    def idna_encode(name):
+        """
+        Borrowed wholesale from the Python Cryptography Project. It turns out
+        that we can't just safely call `idna.encode`: it can explode for
+        wildcard names. This avoids that problem.
+        """
+        for prefix in [u'*.', u'.']:
+            if name.startswith(prefix):
+                name = name[len(prefix):]
+                return prefix.encode('ascii') + idna.encode(name)
+        return idna.encode(name)
+
+    name = idna_encode(name)
+    if sys.version_info >= (3, 0):
+        name = name.decode('utf-8')
+    return name
 
 
-# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
 def get_subj_alt_name(peer_cert):
-    # Search through extensions
-    dns_name = []
-    if not SUBJ_ALT_NAME_SUPPORT:
-        return dns_name
+    """
+    Given an PyOpenSSL certificate, provides all the subject alternative names.
+    """
+    # Pass the cert to cryptography, which has much better APIs for this.
+    # This is technically using private APIs, but should work across all
+    # relevant versions until PyOpenSSL gets something proper for this.
+    cert = _Certificate(openssl_backend, peer_cert._x509)
 
-    general_names = SubjectAltName()
-    for i in range(peer_cert.get_extension_count()):
-        ext = peer_cert.get_extension(i)
-        ext_name = ext.get_short_name()
-        if ext_name != b'subjectAltName':
-            continue
+    # We want to find the SAN extension. Ask Cryptography to locate it (it's
+    # faster than looping in Python)
+    try:
+        ext = cert.extensions.get_extension_for_class(
+            x509.SubjectAlternativeName
+        ).value
+    except x509.ExtensionNotFound:
+        # No such extension, return the empty list.
+        return []
+    except (x509.DuplicateExtension, x509.UnsupportedExtension,
+            x509.UnsupportedGeneralNameType, UnicodeError) as e:
+        # A problem has been found with the quality of the certificate. Assume
+        # no SAN field is present.
+        log.warning(
+            "A problem was encountered with the certificate that prevented "
+            "urllib3 from finding the SubjectAlternativeName field. This can "
+            "affect certificate validation. The error was %s",
+            e,
+        )
+        return []
 
-        # PyOpenSSL returns extension data in ASN.1 encoded form
-        ext_dat = ext.get_data()
-        decoded_dat = der_decoder.decode(ext_dat,
-                                         asn1Spec=general_names)
+    # We want to return dNSName and iPAddress fields. We need to cast the IPs
+    # back to strings because the match_hostname function wants them as
+    # strings.
+    # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8
+    # decoded. This is pretty frustrating, but that's what the standard library
+    # does with certificates, and so we need to attempt to do the same.
+    names = [
+        ('DNS', _dnsname_to_stdlib(name))
+        for name in ext.get_values_for_type(x509.DNSName)
+    ]
+    names.extend(
+        ('IP Address', str(name))
+        for name in ext.get_values_for_type(x509.IPAddress)
+    )
 
-        for name in decoded_dat:
-            if not isinstance(name, SubjectAltName):
-                continue
-            for entry in range(len(name)):
-                component = name.getComponentByPosition(entry)
-                if component.getName() != 'dNSName':
-                    continue
-                dns_name.append(str(component.getComponent()))
-
-    return dns_name
+    return names
 
 
 class WrappedSocket(object):
@@ -282,10 +321,7 @@ class WrappedSocket(object):
             'subject': (
                 (('commonName', x509.get_subject().CN),),
             ),
-            'subjectAltName': [
-                ('DNS', value)
-                for value in get_subj_alt_name(x509)
-            ]
+            'subjectAltName': get_subj_alt_name(x509)
         }
 
     def _reuse(self):
@@ -308,51 +344,88 @@ else:  # Platform-specific: Python 3
 WrappedSocket.makefile = makefile
 
 
+class PyOpenSSLContext(object):
+    """
+    I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible
+    for translating the interface of the standard library ``SSLContext`` object
+    to calls into PyOpenSSL.
+    """
+    def __init__(self, protocol):
+        self.protocol = _openssl_versions[protocol]
+        self._ctx = OpenSSL.SSL.Context(self.protocol)
+        self._options = 0
+        self.check_hostname = False
+
+    @property
+    def options(self):
+        return self._options
+
+    @options.setter
+    def options(self, value):
+        self._options = value
+        self._ctx.set_options(value)
+
+    @property
+    def verify_mode(self):
+        return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()]
+
+    @verify_mode.setter
+    def verify_mode(self, value):
+        self._ctx.set_verify(
+            _stdlib_to_openssl_verify[value],
+            _verify_callback
+        )
+
+    def set_default_verify_paths(self):
+        self._ctx.set_default_verify_paths()
+
+    def set_ciphers(self, ciphers):
+        if isinstance(ciphers, six.text_type):
+            ciphers = ciphers.encode('utf-8')
+        self._ctx.set_cipher_list(ciphers)
+
+    def load_verify_locations(self, cafile=None, capath=None, cadata=None):
+        if cafile is not None:
+            cafile = cafile.encode('utf-8')
+        if capath is not None:
+            capath = capath.encode('utf-8')
+        self._ctx.load_verify_locations(cafile, capath)
+        if cadata is not None:
+            self._ctx.load_verify_locations(BytesIO(cadata))
+
+    def load_cert_chain(self, certfile, keyfile=None, password=None):
+        self._ctx.use_certificate_file(certfile)
+        if password is not None:
+            self._ctx.set_passwd_cb(lambda max_length, prompt_twice, userdata: password)
+        self._ctx.use_privatekey_file(keyfile or certfile)
+
+    def wrap_socket(self, sock, server_side=False,
+                    do_handshake_on_connect=True, suppress_ragged_eofs=True,
+                    server_hostname=None):
+        cnx = OpenSSL.SSL.Connection(self._ctx, sock)
+
+        if isinstance(server_hostname, six.text_type):  # Platform-specific: Python 3
+            server_hostname = server_hostname.encode('utf-8')
+
+        if server_hostname is not None:
+            cnx.set_tlsext_host_name(server_hostname)
+
+        cnx.set_connect_state()
+
+        while True:
+            try:
+                cnx.do_handshake()
+            except OpenSSL.SSL.WantReadError:
+                rd, _, _ = select.select([sock], [], [], sock.gettimeout())
+                if not rd:
+                    raise timeout('select timed out')
+                continue
+            except OpenSSL.SSL.Error as e:
+                raise ssl.SSLError('bad handshake: %r' % e)
+            break
+
+        return WrappedSocket(cnx, sock)
+
+
 def _verify_callback(cnx, x509, err_no, err_depth, return_code):
     return err_no == 0
-
-
-def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
-                    ca_certs=None, server_hostname=None,
-                    ssl_version=None, ca_cert_dir=None):
-    ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version])
-    if certfile:
-        keyfile = keyfile or certfile  # Match behaviour of the normal python ssl library
-        ctx.use_certificate_file(certfile)
-    if keyfile:
-        ctx.use_privatekey_file(keyfile)
-    if cert_reqs != ssl.CERT_NONE:
-        ctx.set_verify(_openssl_verify[cert_reqs], _verify_callback)
-    if ca_certs or ca_cert_dir:
-        try:
-            ctx.load_verify_locations(ca_certs, ca_cert_dir)
-        except OpenSSL.SSL.Error as e:
-            raise ssl.SSLError('bad ca_certs: %r' % ca_certs, e)
-    else:
-        ctx.set_default_verify_paths()
-
-    # Disable TLS compression to mitigate CRIME attack (issue #309)
-    OP_NO_COMPRESSION = 0x20000
-    ctx.set_options(OP_NO_COMPRESSION)
-
-    # Set list of supported ciphersuites.
-    ctx.set_cipher_list(DEFAULT_SSL_CIPHER_LIST)
-
-    cnx = OpenSSL.SSL.Connection(ctx, sock)
-    if isinstance(server_hostname, six.text_type):  # Platform-specific: Python 3
-        server_hostname = server_hostname.encode('utf-8')
-    cnx.set_tlsext_host_name(server_hostname)
-    cnx.set_connect_state()
-    while True:
-        try:
-            cnx.do_handshake()
-        except OpenSSL.SSL.WantReadError:
-            rd, _, _ = select.select([sock], [], [], sock.gettimeout())
-            if not rd:
-                raise timeout('select timed out')
-            continue
-        except OpenSSL.SSL.Error as e:
-            raise ssl.SSLError('bad handshake: %r' % e)
-        break
-
-    return WrappedSocket(cnx, sock)

requests/packages/urllib3/contrib/socks.py

@@ -1,17 +1,23 @@
 # -*- coding: utf-8 -*-
 """
-SOCKS support for urllib3
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This contrib module contains provisional support for SOCKS proxies from within
+This module contains provisional support for SOCKS proxies from within
 urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and
 SOCKS5. To enable its functionality, either install PySocks or install this
 module with the ``socks`` extra.
 
+The SOCKS implementation supports the full range of urllib3 features. It also
+supports the following SOCKS features:
+
+- SOCKS4
+- SOCKS4a
+- SOCKS5
+- Usernames and passwords for the SOCKS proxy
+
 Known Limitations:
 
 - Currently PySocks does not support contacting remote websites via literal
-  IPv6 addresses. Any such connection attempt will fail.
+  IPv6 addresses. Any such connection attempt will fail. You must use a domain
+  name.
 - Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any
   such connection attempt will fail.
 """

requests/packages/urllib3/exceptions.py

@@ -1,4 +1,7 @@
 from __future__ import absolute_import
+from .packages.six.moves.http_client import (
+    IncompleteRead as httplib_IncompleteRead
+)
 # Base Exceptions
 
 
@@ -193,6 +196,35 @@ class ResponseNotChunked(ProtocolError, ValueError):
     pass
 
 
+class BodyNotHttplibCompatible(HTTPError):
+    """
+    Body should be httplib.HTTPResponse like (have an fp attribute which
+    returns raw chunks) for read_chunked().
+    """
+    pass
+
+
+class IncompleteRead(HTTPError, httplib_IncompleteRead):
+    """
+    Response length doesn't match expected Content-Length
+
+    Subclass of http_client.IncompleteRead to allow int value
+    for `partial` to avoid creating large objects on streamed
+    reads.
+    """
+    def __init__(self, partial, expected):
+        super(IncompleteRead, self).__init__(partial, expected)
+
+    def __repr__(self):
+        return ('IncompleteRead(%i bytes read, '
+                '%i more expected)' % (self.partial, self.expected))
+
+
+class InvalidHeader(HTTPError):
+    "The header provided was somehow invalid."
+    pass
+
+
 class ProxySchemeUnknown(AssertionError, ValueError):
     "ProxyManager does not support the supplied scheme"
     # TODO(t-8ch): Stop inheriting from AssertionError in v2.0.

requests/packages/urllib3/fields.py

@@ -130,7 +130,7 @@ class RequestField(object):
             iterable = header_parts.items()
 
         for name, value in iterable:
-            if value:
+            if value is not None:
                 parts.append(self._render_part(name, value))
 
         return '; '.join(parts)

requests/packages/urllib3/filepost.py

@@ -13,7 +13,7 @@ writer = codecs.lookup('utf-8')[3]
 
 def choose_boundary():
     """
-    Our embarassingly-simple replacement for mimetools.choose_boundary.
+    Our embarrassingly-simple replacement for mimetools.choose_boundary.
     """
     return uuid4().hex
 

requests/packages/urllib3/packages/ssl_match_hostname/.gitignore

@@ -1 +0,0 @@
-env

requests/packages/urllib3/packages/ssl_match_hostname/__init__.py

@@ -1,5 +1,11 @@
+import sys
+
 try:
-    # Python 3.2+
+    # Our match_hostname function is the same as 3.5's, so we only want to
+    # import the match_hostname function if it's at least that good.
+    if sys.version_info < (3, 5):
+        raise ImportError("Fallback to vendored code")
+
     from ssl import CertificateError, match_hostname
 except ImportError:
     try:

requests/packages/urllib3/packages/ssl_match_hostname/_implementation.py

@@ -4,8 +4,20 @@
 # stdlib.   http://docs.python.org/3/license.html
 
 import re
+import sys
+
+# ipaddress has been backported to 2.6+ in pypi.  If it is installed on the
+# system, use it to handle IPAddress ServerAltnames (this was added in
+# python-3.5) otherwise only do DNS matching.  This allows
+# backports.ssl_match_hostname to continue to be used all the way back to
+# python-2.4.
+try:
+    import ipaddress
+except ImportError:
+    ipaddress = None
+
+__version__ = '3.5.0.1'
 
-__version__ = '3.4.0.2'
 
 class CertificateError(ValueError):
     pass
@@ -64,6 +76,23 @@ def _dnsname_match(dn, hostname, max_wildcards=1):
     return pat.match(hostname)
 
 
+def _to_unicode(obj):
+    if isinstance(obj, str) and sys.version_info < (3,):
+        obj = unicode(obj, encoding='ascii', errors='strict')
+    return obj
+
+def _ipaddress_match(ipname, host_ip):
+    """Exact matching of IP addresses.
+
+    RFC 6125 explicitly doesn't define an algorithm for this
+    (section 1.7.2 - "Out of Scope").
+    """
+    # OpenSSL may add a trailing newline to a subjectAltName's IP address
+    # Divergence from upstream: ipaddress can't handle byte str
+    ip = ipaddress.ip_address(_to_unicode(ipname).rstrip())
+    return ip == host_ip
+
+
 def match_hostname(cert, hostname):
     """Verify that *cert* (in decoded format as returned by
     SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 and RFC 6125
@@ -73,12 +102,35 @@ def match_hostname(cert, hostname):
     returns nothing.
     """
     if not cert:
-        raise ValueError("empty or no certificate")
+        raise ValueError("empty or no certificate, match_hostname needs a "
+                         "SSL socket or SSL context with either "
+                         "CERT_OPTIONAL or CERT_REQUIRED")
+    try:
+        # Divergence from upstream: ipaddress can't handle byte str
+        host_ip = ipaddress.ip_address(_to_unicode(hostname))
+    except ValueError:
+        # Not an IP address (common case)
+        host_ip = None
+    except UnicodeError:
+        # Divergence from upstream: Have to deal with ipaddress not taking
+        # byte strings.  addresses should be all ascii, so we consider it not
+        # an ipaddress in this case
+        host_ip = None
+    except AttributeError:
+        # Divergence from upstream: Make ipaddress library optional
+        if ipaddress is None:
+            host_ip = None
+        else:
+            raise
     dnsnames = []
     san = cert.get('subjectAltName', ())
     for key, value in san:
         if key == 'DNS':
-            if _dnsname_match(value, hostname):
+            if host_ip is None and _dnsname_match(value, hostname):
+                return
+            dnsnames.append(value)
+        elif key == 'IP Address':
+            if host_ip is not None and _ipaddress_match(value, host_ip):
                 return
             dnsnames.append(value)
     if not dnsnames:

requests/packages/urllib3/poolmanager.py

@@ -3,15 +3,11 @@ import collections
 import functools
 import logging
 
-try:  # Python 3
-    from urllib.parse import urljoin
-except ImportError:
-    from urlparse import urljoin
-
 from ._collections import RecentlyUsedContainer
 from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
 from .connectionpool import port_by_scheme
 from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown
+from .packages.six.moves.urllib.parse import urljoin
 from .request import RequestMethods
 from .util.url import parse_url
 from .util.retry import Retry
@@ -23,7 +19,7 @@ __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
 log = logging.getLogger(__name__)
 
 SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
-                'ssl_version', 'ca_cert_dir')
+                'ssl_version', 'ca_cert_dir', 'ssl_context')
 
 # The base fields to use when determining what pool to get a connection from;
 # these do not rely on the ``connection_pool_kw`` and can be determined by the

requests/packages/urllib3/request.py

@@ -1,10 +1,7 @@
 from __future__ import absolute_import
-try:
-    from urllib.parse import urlencode
-except ImportError:
-    from urllib import urlencode
 
 from .filepost import encode_multipart_formdata
+from .packages.six.moves.urllib.parse import urlencode
 
 
 __all__ = ['RequestMethods']

requests/packages/urllib3/response.py

@@ -2,18 +2,22 @@ from __future__ import absolute_import
 from contextlib import contextmanager
 import zlib
 import io
+import logging
 from socket import timeout as SocketTimeout
 from socket import error as SocketError
 
 from ._collections import HTTPHeaderDict
 from .exceptions import (
-    ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked
+    BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError,
+    ResponseNotChunked, IncompleteRead, InvalidHeader
 )
 from .packages.six import string_types as basestring, binary_type, PY3
 from .packages.six.moves import http_client as httplib
 from .connection import HTTPException, BaseSSLError
 from .util.response import is_fp_closed, is_response_to_head
 
+log = logging.getLogger(__name__)
+
 
 class DeflateDecoder(object):
 
@@ -89,6 +93,14 @@ class HTTPResponse(io.IOBase):
         When this HTTPResponse wrapper is generated from an httplib.HTTPResponse
         object, it's convenient to include the original for debug purposes. It's
         otherwise unused.
+
+    :param retries:
+        The retries contains the last :class:`~urllib3.util.retry.Retry` that
+        was used during the request.
+
+    :param enforce_content_length:
+        Enforce content length checking. Body returned by server must match
+        value of Content-Length header, if present. Otherwise, raise error.
     """
 
     CONTENT_DECODERS = ['gzip', 'deflate']
@@ -96,7 +108,8 @@ class HTTPResponse(io.IOBase):
 
     def __init__(self, body='', headers=None, status=0, version=0, reason=None,
                  strict=0, preload_content=True, decode_content=True,
-                 original_response=None, pool=None, connection=None):
+                 original_response=None, pool=None, connection=None,
+                 retries=None, enforce_content_length=False, request_method=None):
 
         if isinstance(headers, HTTPHeaderDict):
             self.headers = headers
@@ -107,6 +120,8 @@ class HTTPResponse(io.IOBase):
         self.reason = reason
         self.strict = strict
         self.decode_content = decode_content
+        self.retries = retries
+        self.enforce_content_length = enforce_content_length
 
         self._decoder = None
         self._body = None
@@ -132,6 +147,9 @@ class HTTPResponse(io.IOBase):
         if "chunked" in encodings:
             self.chunked = True
 
+        # Determine length of response
+        self.length_remaining = self._init_length(request_method)
+
         # If requested, preload the body.
         if preload_content and not self._body:
             self._body = self.read(decode_content=decode_content)
@@ -177,9 +195,57 @@ class HTTPResponse(io.IOBase):
         """
         return self._fp_bytes_read
 
+    def _init_length(self, request_method):
+        """
+        Set initial length value for Response content if available.
+        """
+        length = self.headers.get('content-length')
+
+        if length is not None and self.chunked:
+            # This Response will fail with an IncompleteRead if it can't be
+            # received as chunked. This method falls back to attempt reading
+            # the response before raising an exception.
+            log.warning("Received response with both Content-Length and "
+                        "Transfer-Encoding set. This is expressly forbidden "
+                        "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
+                        "attempting to process response as Transfer-Encoding: "
+                        "chunked.")
+            return None
+
+        elif length is not None:
+            try:
+                # RFC 7230 section 3.3.2 specifies multiple content lengths can
+                # be sent in a single Content-Length header
+                # (e.g. Content-Length: 42, 42). This line ensures the values
+                # are all valid ints and that as long as the `set` length is 1,
+                # all values are the same. Otherwise, the header is invalid.
+                lengths = set([int(val) for val in length.split(',')])
+                if len(lengths) > 1:
+                    raise InvalidHeader("Content-Length contained multiple "
+                                        "unmatching values (%s)" % length)
+                length = lengths.pop()
+            except ValueError:
+                length = None
+            else:
+                if length < 0:
+                    length = None
+
+        # Convert status to int for comparison
+        # In some cases, httplib returns a status of "_UNKNOWN"
+        try:
+            status = int(self.status)
+        except ValueError:
+            status = 0
+
+        # Check for responses that shouldn't include a body
+        if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD':
+            length = 0
+
+        return length
+
     def _init_decoder(self):
         """
-        Set-up the _decoder attribute if necessar.
+        Set-up the _decoder attribute if necessary.
         """
         # Note: content-encoding value should be case-insensitive, per RFC 7230
         # Section 3.2
@@ -322,9 +388,18 @@ class HTTPResponse(io.IOBase):
                     # no harm in redundantly calling close.
                     self._fp.close()
                     flush_decoder = True
+                    if self.enforce_content_length and self.length_remaining not in (0, None):
+                        # This is an edge case that httplib failed to cover due
+                        # to concerns of backward compatibility. We're
+                        # addressing it here to make sure IncompleteRead is
+                        # raised during streaming, so all calls with incorrect
+                        # Content-Length are caught.
+                        raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
 
         if data:
             self._fp_bytes_read += len(data)
+            if self.length_remaining is not None:
+                self.length_remaining -= len(data)
 
             data = self._decode(data, decode_content, flush_decoder)
 
@@ -349,7 +424,7 @@ class HTTPResponse(io.IOBase):
             If True, will attempt to decode the body based on the
             'content-encoding' header.
         """
-        if self.chunked:
+        if self.chunked and self.supports_chunked_reads():
             for line in self.read_chunked(amt, decode_content=decode_content):
                 yield line
         else:
@@ -407,10 +482,10 @@ class HTTPResponse(io.IOBase):
     def closed(self):
         if self._fp is None:
             return True
+        elif hasattr(self._fp, 'isclosed'):
+            return self._fp.isclosed()
         elif hasattr(self._fp, 'closed'):
             return self._fp.closed
-        elif hasattr(self._fp, 'isclosed'):  # Python 2
-            return self._fp.isclosed()
         else:
             return True
 
@@ -440,6 +515,15 @@ class HTTPResponse(io.IOBase):
             b[:len(temp)] = temp
             return len(temp)
 
+    def supports_chunked_reads(self):
+        """
+        Checks if the underlying file-like object looks like a
+        httplib.HTTPResponse object. We do this by testing for the fp
+        attribute. If it is present we assume it returns raw chunks as
+        processed by read_chunked().
+        """
+        return hasattr(self._fp, 'fp')
+
     def _update_chunk_length(self):
         # First, we'll figure out length of a chunk and then
         # we'll try to read it from socket.
@@ -491,6 +575,10 @@ class HTTPResponse(io.IOBase):
             raise ResponseNotChunked(
                 "Response is not chunked. "
                 "Header 'transfer-encoding: chunked' is missing.")
+        if not self.supports_chunked_reads():
+            raise BodyNotHttplibCompatible(
+                "Body should be httplib.HTTPResponse like. "
+                "It should have have an fp attribute which returns raw chunks.")
 
         # Don't bother reading the body of a HEAD request.
         if self._original_response and is_response_to_head(self._original_response):

requests/packages/urllib3/util/response.py

@@ -12,6 +12,13 @@ def is_fp_closed(obj):
         The file-like object to check.
     """
 
+    try:
+        # Check `isclosed()` first, in case Python3 doesn't set `closed`.
+        # GH Issue #928
+        return obj.isclosed()
+    except AttributeError:
+        pass
+
     try:
         # Check via the official file-like-object way.
         return obj.closed

requests/packages/urllib3/util/retry.py

@@ -1,6 +1,10 @@
 from __future__ import absolute_import
 import time
 import logging
+from collections import namedtuple
+from itertools import takewhile
+import email
+import re
 
 from ..exceptions import (
     ConnectTimeoutError,
@@ -8,12 +12,17 @@ from ..exceptions import (
     ProtocolError,
     ReadTimeoutError,
     ResponseError,
+    InvalidHeader,
 )
 from ..packages import six
 
 
 log = logging.getLogger(__name__)
 
+# Data structure for representing the metadata of requests that result in a retry.
+RequestHistory = namedtuple('RequestHistory', ["method", "url", "error",
+                                               "status", "redirect_location"])
+
 
 class Retry(object):
     """ Retry configuration.
@@ -113,18 +122,29 @@ class Retry(object):
         whether we should raise an exception, or return a response,
         if status falls in ``status_forcelist`` range and retries have
         been exhausted.
+
+    :param tuple history: The history of the request encountered during
+        each call to :meth:`~Retry.increment`. The list is in the order
+        the requests occurred. Each list item is of class :class:`RequestHistory`.
+
+    :param bool respect_retry_after_header:
+        Whether to respect Retry-After header on status codes defined as
+        :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
+
     """
 
     DEFAULT_METHOD_WHITELIST = frozenset([
         'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'])
 
+    RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
+
     #: Maximum backoff time.
     BACKOFF_MAX = 120
 
     def __init__(self, total=10, connect=None, read=None, redirect=None,
                  method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None,
                  backoff_factor=0, raise_on_redirect=True, raise_on_status=True,
-                 _observed_errors=0):
+                 history=None, respect_retry_after_header=True):
 
         self.total = total
         self.connect = connect
@@ -140,7 +160,8 @@ class Retry(object):
         self.backoff_factor = backoff_factor
         self.raise_on_redirect = raise_on_redirect
         self.raise_on_status = raise_on_status
-        self._observed_errors = _observed_errors  # TODO: use .history instead?
+        self.history = history or tuple()
+        self.respect_retry_after_header = respect_retry_after_header
 
     def new(self, **kw):
         params = dict(
@@ -151,7 +172,7 @@ class Retry(object):
             backoff_factor=self.backoff_factor,
             raise_on_redirect=self.raise_on_redirect,
             raise_on_status=self.raise_on_status,
-            _observed_errors=self._observed_errors,
+            history=self.history,
         )
         params.update(kw)
         return type(self)(**params)
@@ -175,23 +196,71 @@ class Retry(object):
 
         :rtype: float
         """
-        if self._observed_errors <= 1:
+        # We want to consider only the last consecutive errors sequence (Ignore redirects).
+        consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None,
+                                                    reversed(self.history))))
+        if consecutive_errors_len <= 1:
             return 0
 
-        backoff_value = self.backoff_factor * (2 ** (self._observed_errors - 1))
+        backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
         return min(self.BACKOFF_MAX, backoff_value)
 
-    def sleep(self):
-        """ Sleep between retry attempts using an exponential backoff.
+    def parse_retry_after(self, retry_after):
+        # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
+        if re.match(r"^\s*[0-9]+\s*$", retry_after):
+            seconds = int(retry_after)
+        else:
+            retry_date_tuple = email.utils.parsedate(retry_after)
+            if retry_date_tuple is None:
+                raise InvalidHeader("Invalid Retry-After header: %s" % retry_after)
+            retry_date = time.mktime(retry_date_tuple)
+            seconds = retry_date - time.time()
 
-        By default, the backoff factor is 0 and this method will return
-        immediately.
-        """
+        if seconds < 0:
+            seconds = 0
+
+        return seconds
+
+    def get_retry_after(self, response):
+        """ Get the value of Retry-After in seconds. """
+
+        retry_after = response.getheader("Retry-After")
+
+        if retry_after is None:
+            return None
+
+        return self.parse_retry_after(retry_after)
+
+    def sleep_for_retry(self, response=None):
+        retry_after = self.get_retry_after(response)
+        if retry_after:
+            time.sleep(retry_after)
+            return True
+
+        return False
+
+    def _sleep_backoff(self):
         backoff = self.get_backoff_time()
         if backoff <= 0:
             return
         time.sleep(backoff)
 
+    def sleep(self, response=None):
+        """ Sleep between retry attempts.
+
+        This method will respect a server's ``Retry-After`` response header
+        and sleep the duration of the time requested. If that is not present, it
+        will use an exponential backoff. By default, the backoff factor is 0 and
+        this method will return immediately.
+        """
+
+        if response:
+            slept = self.sleep_for_retry(response)
+            if slept:
+                return
+
+        self._sleep_backoff()
+
     def _is_connection_error(self, err):
         """ Errors when we're fairly sure that the server did not receive the
         request, so it should be safe to retry.
@@ -204,13 +273,17 @@ class Retry(object):
         """
         return isinstance(err, (ReadTimeoutError, ProtocolError))
 
-    def is_forced_retry(self, method, status_code):
+    def is_retry(self, method, status_code, has_retry_after=False):
         """ Is this method/status code retryable? (Based on method/codes whitelists)
         """
         if self.method_whitelist and method.upper() not in self.method_whitelist:
             return False
 
-        return self.status_forcelist and status_code in self.status_forcelist
+        if self.status_forcelist and status_code in self.status_forcelist:
+            return True
+
+        return (self.total and self.respect_retry_after_header and
+                has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES))
 
     def is_exhausted(self):
         """ Are we out of retries? """
@@ -241,11 +314,12 @@ class Retry(object):
         if total is not None:
             total -= 1
 
-        _observed_errors = self._observed_errors
         connect = self.connect
         read = self.read
         redirect = self.redirect
         cause = 'unknown'
+        status = None
+        redirect_location = None
 
         if error and self._is_connection_error(error):
             # Connect retry?
@@ -253,7 +327,6 @@ class Retry(object):
                 raise six.reraise(type(error), error, _stacktrace)
             elif connect is not None:
                 connect -= 1
-            _observed_errors += 1
 
         elif error and self._is_read_error(error):
             # Read retry?
@@ -261,27 +334,30 @@ class Retry(object):
                 raise six.reraise(type(error), error, _stacktrace)
             elif read is not None:
                 read -= 1
-            _observed_errors += 1
 
         elif response and response.get_redirect_location():
             # Redirect retry?
             if redirect is not None:
                 redirect -= 1
             cause = 'too many redirects'
+            redirect_location = response.get_redirect_location()
+            status = response.status
 
         else:
             # Incrementing because of a server error like a 500 in
             # status_forcelist and a the given method is in the whitelist
-            _observed_errors += 1
             cause = ResponseError.GENERIC_ERROR
             if response and response.status:
                 cause = ResponseError.SPECIFIC_ERROR.format(
                     status_code=response.status)
+                status = response.status
+
+        history = self.history + (RequestHistory(method, url, error, status, redirect_location),)
 
         new_retry = self.new(
             total=total,
             connect=connect, read=read, redirect=redirect,
-            _observed_errors=_observed_errors)
+            history=history)
 
         if new_retry.is_exhausted():
             raise MaxRetryError(_pool, url, error or ResponseError(cause))

requests/packages/urllib3/util/ssl_.py

@@ -11,7 +11,6 @@ from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning
 
 SSLContext = None
 HAS_SNI = False
-create_default_context = None
 IS_PYOPENSSL = False
 
 # Maps the length of a digest to a possible hash function producing this digest
@@ -63,14 +62,25 @@ except ImportError:
 # The general intent is:
 # - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE),
 # - prefer ECDHE over DHE for better performance,
-# - prefer any AES-GCM over any AES-CBC for better performance and security,
-# - use 3DES as fallback which is secure but slow,
+# - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and
+#   security,
+# - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common,
 # - disable NULL authentication, MD5 MACs and DSS for security reasons.
-DEFAULT_CIPHERS = (
-    'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
-    'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:'
-    '!eNULL:!MD5'
-)
+DEFAULT_CIPHERS = ':'.join([
+    'ECDH+AESGCM',
+    'ECDH+CHACHA20',
+    'DH+AESGCM',
+    'DH+CHACHA20',
+    'ECDH+AES256',
+    'DH+AES256',
+    'ECDH+AES128',
+    'DH+AES',
+    'RSA+AESGCM',
+    'RSA+AES',
+    '!aNULL',
+    '!eNULL',
+    '!MD5',
+])
 
 try:
     from ssl import SSLContext  # Modern SSL?
@@ -117,8 +127,8 @@ except ImportError:
                 'urllib3 from configuring SSL appropriately and may cause '
                 'certain SSL connections to fail. You can upgrade to a newer '
                 'version of Python to solve this. For more information, see '
-                'https://urllib3.readthedocs.io/en/latest/security.html'
-                '#insecureplatformwarning.',
+                'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
+                '#ssl-warnings',
                 InsecurePlatformWarning
             )
             kwargs = {
@@ -287,6 +297,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
     """
     context = ssl_context
     if context is None:
+        # Note: This branch of code and all the variables in it are no longer
+        # used by urllib3 itself. We should consider deprecating and removing
+        # this code.
         context = create_urllib3_context(ssl_version, cert_reqs,
                                          ciphers=ciphers)
 
@@ -301,6 +314,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
             if e.errno == errno.ENOENT:
                 raise SSLError(e)
             raise
+    elif getattr(context, 'load_default_certs', None) is not None:
+        # try to load OS default certs; works well on Windows (require Python3.4+)
+        context.load_default_certs()
 
     if certfile:
         context.load_cert_chain(certfile, keyfile)
@@ -313,8 +329,8 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
         'This may cause the server to present an incorrect TLS '
         'certificate, which can cause validation failures. You can upgrade to '
         'a newer version of Python to solve this. For more information, see '
-        'https://urllib3.readthedocs.io/en/latest/security.html'
-        '#snimissingwarning.',
+        'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
+        '#ssl-warnings',
         SNIMissingWarning
     )
     return context.wrap_socket(sock)

requests/packages/urllib3/util/timeout.py

@@ -111,8 +111,8 @@ class Timeout(object):
         :param name: The name of the timeout attribute to validate. This is
             used to specify in error messages.
         :return: The validated and casted version of the given value.
-        :raises ValueError: If the type is not an integer or a float, or if it
-            is a numeric value less than zero.
+        :raises ValueError: If it is a numeric value less than or equal to
+            zero, or the type is not an integer, float, or None.
         """
         if value is _Default:
             return cls.DEFAULT_TIMEOUT
@@ -120,20 +120,23 @@ class Timeout(object):
         if value is None or value is cls.DEFAULT_TIMEOUT:
             return value
 
+        if isinstance(value, bool):
+            raise ValueError("Timeout cannot be a boolean value. It must "
+                             "be an int, float or None.")
         try:
             float(value)
         except (TypeError, ValueError):
             raise ValueError("Timeout value %s was %s, but it must be an "
-                             "int or float." % (name, value))
+                             "int, float or None." % (name, value))
 
         try:
-            if value < 0:
+            if value <= 0:
                 raise ValueError("Attempted to set %s timeout to %s, but the "
                                  "timeout cannot be set to a value less "
-                                 "than 0." % (name, value))
+                                 "than or equal to 0." % (name, value))
         except TypeError:  # Python 3
             raise ValueError("Timeout value %s was %s, but it must be an "
-                             "int or float." % (name, value))
+                             "int, float or None." % (name, value))
 
         return value
 

requests/packages/urllib3/util/url.py

@@ -10,14 +10,19 @@ url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment']
 class Url(namedtuple('Url', url_attrs)):
     """
     Datastructure for representing an HTTP URL. Used as a return value for
-    :func:`parse_url`.
+    :func:`parse_url`. Both the scheme and host are normalized as they are
+    both case-insensitive according to RFC 3986.
     """
-    slots = ()
+    __slots__ = ()
 
     def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None,
                 query=None, fragment=None):
         if path and not path.startswith('/'):
             path = '/' + path
+        if scheme:
+            scheme = scheme.lower()
+        if host:
+            host = host.lower()
         return super(Url, cls).__new__(cls, scheme, auth, host, port, path,
                                        query, fragment)
 
@@ -184,10 +189,14 @@ def parse_url(url):
             host = _host
 
         if port:
-            # If given, ports must be integers.
+            # If given, ports must be integers. No whitespace, no plus or
+            # minus prefixes, no non-integer digits such as ^2 (superscript).
             if not port.isdigit():
                 raise LocationParseError(url)
-            port = int(port)
+            try:
+                port = int(port)
+            except ValueError:
+                raise LocationParseError(url)
         else:
             # Blank ports are cool, too. (rfc3986#section-3.2.3)
             port = None
@@ -211,7 +220,7 @@ def parse_url(url):
 
 def get_host(url):
     """
-    Deprecated. Use :func:`.parse_url` instead.
+    Deprecated. Use :func:`parse_url` instead.
     """
     p = parse_url(url)
     return p.scheme or 'http', p.hostname, p.port