Merge pull request #1128 from mvid/master

Updating urllib3 to current master

requests/packages/urllib3/connectionpool.py

@@ -9,6 +9,7 @@ import socket
 import errno
 
 from socket import error as SocketError, timeout as SocketTimeout
+from .util import resolve_cert_reqs, resolve_ssl_version
 
 try: # Python 3
     from http.client import HTTPConnection, HTTPException
@@ -80,31 +81,29 @@ class VerifiedHTTPSConnection(HTTPSConnection):
     ssl_version = None
 
     def set_cert(self, key_file=None, cert_file=None,
-                 cert_reqs='CERT_NONE', ca_certs=None):
-        ssl_req_scheme = {
-            'CERT_NONE': ssl.CERT_NONE,
-            'CERT_OPTIONAL': ssl.CERT_OPTIONAL,
-            'CERT_REQUIRED': ssl.CERT_REQUIRED
-        }
+                 cert_reqs=None, ca_certs=None):
 
         self.key_file = key_file
         self.cert_file = cert_file
-        self.cert_reqs = ssl_req_scheme.get(cert_reqs) or ssl.CERT_NONE
+        self.cert_reqs = cert_reqs
         self.ca_certs = ca_certs
 
     def connect(self):
         # Add certificate verification
         sock = socket.create_connection((self.host, self.port), self.timeout)
 
+        resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs)
+        resolved_ssl_version = resolve_ssl_version(self.ssl_version)
+
         # Wrap socket using verification with the root certs in
         # trusted_root_certs
         self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file,
-                                    cert_reqs=self.cert_reqs,
+                                    cert_reqs=resolved_cert_reqs,
                                     ca_certs=self.ca_certs,
                                     server_hostname=self.host,
-                                    ssl_version=self.ssl_version)
+                                    ssl_version=resolved_ssl_version)
 
-        if self.ca_certs:
+        if resolved_cert_reqs != ssl.CERT_NONE:
             match_hostname(self.sock.getpeercert(), self.host)
 
 
@@ -514,7 +513,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
                  strict=False, timeout=None, maxsize=1,
                  block=False, headers=None,
                  key_file=None, cert_file=None,
-                 cert_reqs='CERT_NONE', ca_certs=None, ssl_version=None):
+                 cert_reqs=None, ca_certs=None, ssl_version=None):
 
         HTTPConnectionPool.__init__(self, host, port,
                                     strict, timeout, maxsize,
@@ -548,10 +547,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
         connection.set_cert(key_file=self.key_file, cert_file=self.cert_file,
                             cert_reqs=self.cert_reqs, ca_certs=self.ca_certs)
 
-        if self.ssl_version is None:
-            connection.ssl_version = ssl.PROTOCOL_SSLv23
-        else:
-            connection.ssl_version = self.ssl_version
+        connection.ssl_version = self.ssl_version
 
         return connection
 

requests/packages/urllib3/poolmanager.py

@@ -58,6 +58,17 @@ class PoolManager(RequestMethods):
         self.pools = RecentlyUsedContainer(num_pools,
                                            dispose_func=lambda p: p.close())
 
+    def _new_pool(self, scheme, host, port):
+        """
+        Create a new :class:`ConnectionPool` based on host, port and scheme.
+
+        This method is used to actually create the connection pools handed out
+        by :meth:`connection_from_url` and companion methods. It is intended
+        to be overridden for customization.
+        """
+        pool_cls = pool_classes_by_scheme[scheme]
+        return pool_cls(host, port, **self.connection_pool_kw)
+
     def clear(self):
         """
         Empty our store of pools and direct them all to close.
@@ -74,6 +85,7 @@ class PoolManager(RequestMethods):
         If ``port`` isn't given, it will be derived from the ``scheme`` using
         ``urllib3.connectionpool.port_by_scheme``.
         """
+        scheme = scheme or 'http'
         port = port or port_by_scheme.get(scheme, 80)
 
         pool_key = (scheme, host, port)
@@ -85,11 +97,8 @@ class PoolManager(RequestMethods):
             return pool
 
         # Make a fresh ConnectionPool of the desired type
-        pool_cls = pool_classes_by_scheme[scheme]
-        pool = pool_cls(host, port, **self.connection_pool_kw)
-
+        pool = self._new_pool(scheme, host, port)
         self.pools[pool_key] = pool
-
         return pool
 
     def connection_from_url(self, url):

requests/packages/urllib3/response.py

@@ -145,7 +145,17 @@ class HTTPResponse(object):
                 # cStringIO doesn't like amt=None
                 data = self._fp.read()
             else:
-                return self._fp.read(amt)
+                data = self._fp.read(amt)
+                if amt != 0 and not data:  # Platform-specific: Buggy versions of Python.
+                    # Close the connection when no data is returned
+                    #
+                    # This is redundant to what httplib/http.client _should_
+                    # already do.  However, versions of python released before
+                    # December 15, 2012 (http://bugs.python.org/issue16298) do not
+                    # properly close the connection in all cases. There is no harm
+                    # in redundantly calling close.
+                    self._fp.close()
+                return data
 
             try:
                 if decode_content and decoder:

requests/packages/urllib3/util.py

@@ -22,6 +22,7 @@ try:  # Test for SSL features
     SSLContext = None
     HAS_SNI = False
 
+    import ssl
     from ssl import wrap_socket, CERT_NONE, SSLError, PROTOCOL_SSLv23
     from ssl import SSLContext  # Modern SSL?
     from ssl import HAS_SNI  # Has SNI?
@@ -263,10 +264,48 @@ def is_connection_dropped(conn):
             return True
 
 
+def resolve_cert_reqs(candidate):
+    """
+    Resolves the argument to a numeric constant, which can be passed to
+    the wrap_socket function/method from the ssl module.
+    Defaults to :data:`ssl.CERT_NONE`.
+    If given a string it is assumed to be the name of the constant in the
+    :mod:`ssl` module or its abbrevation.
+    (So you can specify `REQUIRED` instead of `CERT_REQUIRED`.
+    If it's neither `None` nor a string we assume it is already the numeric
+    constant which can directly be passed to wrap_socket.
+    """
+    if candidate is None:
+        return CERT_NONE
+
+    if isinstance(candidate, str):
+        res = getattr(ssl, candidate, None)
+        if res is None:
+            res = getattr(ssl, 'CERT_' + candidate)
+        return res
+
+    return candidate
+
+
+def resolve_ssl_version(candidate):
+    """
+    like resolve_cert_reqs
+    """
+    if candidate is None:
+        return PROTOCOL_SSLv23
+
+    if isinstance(candidate, str):
+        res = getattr(ssl, candidate, None)
+        if res is None:
+            res = getattr(ssl, 'PROTOCOL_' + candidate)
+        return res
+
+    return candidate
+
 if SSLContext is not None:  # Python 3.2+
-    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE,
+    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
                         ca_certs=None, server_hostname=None,
-                        ssl_version=PROTOCOL_SSLv23):
+                        ssl_version=None):
         """
         All arguments except `server_hostname` have the same meaning as for
         :func:`ssl.wrap_socket`
@@ -279,8 +318,9 @@ if SSLContext is not None:  # Python 3.2+
         if ca_certs:
             try:
                 context.load_verify_locations(ca_certs)
-            except TypeError as e:  # Reraise as SSLError
-                # FIXME: This block needs a test.
+            # Py32 raises IOError
+            # Py33 raises FileNotFoundError
+            except Exception as e:  # Reraise as SSLError
                 raise SSLError(e)
         if certfile:
             # FIXME: This block needs a test.
@@ -290,9 +330,9 @@ if SSLContext is not None:  # Python 3.2+
         return context.wrap_socket(sock)
 
 else:  # Python 3.1 and earlier
-    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE,
+    def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
                         ca_certs=None, server_hostname=None,
-                        ssl_version=PROTOCOL_SSLv23):
+                        ssl_version=None):
         return wrap_socket(sock, keyfile=keyfile, certfile=certfile,
                            ca_certs=ca_certs, cert_reqs=cert_reqs,
                            ssl_version=ssl_version)