Merge pull request #2179 from kevinburke/urllib3-pypy-ssl

Updates urllib3 to 528ad3c

requests/packages/urllib3/__init__.py

@@ -57,7 +57,7 @@ del NullHandler
 
 # Set security warning to only go off once by default.
 import warnings
-warnings.simplefilter('module', exceptions.InsecureRequestWarning)
+warnings.simplefilter('module', exceptions.SecurityWarning)
 
 def disable_warnings(category=exceptions.HTTPWarning):
     """

requests/packages/urllib3/connection.py

@@ -1,6 +1,8 @@
+import datetime
 import sys
 import socket
 from socket import timeout as SocketTimeout
+import warnings
 
 try:  # Python 3
     from http.client import HTTPConnection as _HTTPConnection, HTTPException
@@ -26,6 +28,7 @@ except (ImportError, AttributeError):  # Platform-specific: No SSL.
 
 from .exceptions import (
     ConnectTimeoutError,
+    SystemTimeWarning,
 )
 from .packages.ssl_match_hostname import match_hostname
 from .packages import six
@@ -45,6 +48,8 @@ port_by_scheme = {
     'https': 443,
 }
 
+RECENT_DATE = datetime.date(2014, 1, 1)
+
 
 class HTTPConnection(_HTTPConnection, object):
     """
@@ -172,6 +177,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
     cert_reqs = None
     ca_certs = None
     ssl_version = None
+    assert_fingerprint = None
 
     def set_cert(self, key_file=None, cert_file=None,
                  cert_reqs=None, ca_certs=None,
@@ -206,6 +212,14 @@ class VerifiedHTTPSConnection(HTTPSConnection):
             # Override the host with the one we're requesting data from.
             hostname = self._tunnel_host
 
+        is_time_off = datetime.date.today() < RECENT_DATE
+        if is_time_off:
+            warnings.warn((
+                'System time is way off (before {0}). This will probably '
+                'lead to SSL verification errors').format(RECENT_DATE),
+                SystemTimeWarning
+            )
+
         # Wrap socket using verification with the root certs in
         # trusted_root_certs
         self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file,
@@ -214,15 +228,16 @@ class VerifiedHTTPSConnection(HTTPSConnection):
                                     server_hostname=hostname,
                                     ssl_version=resolved_ssl_version)
 
-        if resolved_cert_reqs != ssl.CERT_NONE:
-            if self.assert_fingerprint:
-                assert_fingerprint(self.sock.getpeercert(binary_form=True),
-                                   self.assert_fingerprint)
-            elif self.assert_hostname is not False:
-                match_hostname(self.sock.getpeercert(),
-                               self.assert_hostname or hostname)
+        if self.assert_fingerprint:
+            assert_fingerprint(self.sock.getpeercert(binary_form=True),
+                               self.assert_fingerprint)
+        elif resolved_cert_reqs != ssl.CERT_NONE \
+                and self.assert_hostname is not False:
+            match_hostname(self.sock.getpeercert(),
+                           self.assert_hostname or hostname)
 
-        self.is_verified = resolved_cert_reqs == ssl.CERT_REQUIRED
+        self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED
+                            or self.assert_fingerprint is not None)
 
 
 if ssl:

requests/packages/urllib3/connectionpool.py

@@ -718,7 +718,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
         super(HTTPSConnectionPool, self)._validate_conn(conn)
 
         # Force connect early to allow us to validate the connection.
-        if not conn.sock:
+        if not getattr(conn, 'sock', None):  # AppEngine might not have  `.sock`
             conn.connect()
 
         if not conn.is_verified:

requests/packages/urllib3/contrib/pyopenssl.py

@@ -46,8 +46,12 @@ Module Variables
 
 '''
 
-from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
-from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName
+try:
+    from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
+    from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName
+except SyntaxError as e:
+    raise ImportError(e)
+
 import OpenSSL.SSL
 from pyasn1.codec.der import decoder as der_decoder
 from pyasn1.type import univ, constraint
@@ -155,18 +159,24 @@ def get_subj_alt_name(peer_cert):
 
 
 class WrappedSocket(object):
-    '''API-compatibility wrapper for Python OpenSSL's Connection-class.'''
+    '''API-compatibility wrapper for Python OpenSSL's Connection-class.
+
+    Note: _makefile_refs, _drop() and _reuse() are needed for the garbage
+    collector of pypy.
+    '''
 
     def __init__(self, connection, socket, suppress_ragged_eofs=True):
         self.connection = connection
         self.socket = socket
         self.suppress_ragged_eofs = suppress_ragged_eofs
+        self._makefile_refs = 0
 
     def fileno(self):
         return self.socket.fileno()
 
     def makefile(self, mode, bufsize=-1):
-        return _fileobject(self, mode, bufsize)
+        self._makefile_refs += 1
+        return _fileobject(self, mode, bufsize, close=True)
 
     def recv(self, *args, **kwargs):
         try:
@@ -180,7 +190,7 @@ class WrappedSocket(object):
             rd, wd, ed = select.select(
                 [self.socket], [], [], self.socket.gettimeout())
             if not rd:
-                raise timeout()
+                raise timeout('The read operation timed out')
             else:
                 return self.recv(*args, **kwargs)
         else:
@@ -193,7 +203,10 @@ class WrappedSocket(object):
         return self.connection.sendall(data)
 
     def close(self):
-        return self.connection.shutdown()
+        if self._makefile_refs < 1:
+            return self.connection.shutdown()
+        else:
+            self._makefile_refs -= 1
 
     def getpeercert(self, binary_form=False):
         x509 = self.connection.get_peer_certificate()
@@ -216,6 +229,15 @@ class WrappedSocket(object):
             ]
         }
 
+    def _reuse(self):
+        self._makefile_refs += 1
+
+    def _drop(self):
+        if self._makefile_refs < 1:
+            self.close()
+        else:
+            self._makefile_refs -= 1
+
 
 def _verify_callback(cnx, x509, err_no, err_depth, return_code):
     return err_no == 0

requests/packages/urllib3/exceptions.py

@@ -60,7 +60,14 @@ ConnectionError = ProtocolError
 ## Leaf Exceptions
 
 class MaxRetryError(RequestError):
-    "Raised when the maximum number of retries is exceeded."
+    """Raised when the maximum number of retries is exceeded.
+
+    :param pool: The connection pool
+    :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool`
+    :param string url: The requested Url
+    :param exceptions.Exception reason: The underlying error
+
+    """
 
     def __init__(self, pool, url, reason=None):
         self.reason = reason
@@ -134,6 +141,16 @@ class LocationParseError(LocationValueError):
         self.location = location
 
 
-class InsecureRequestWarning(HTTPWarning):
+class SecurityWarning(HTTPWarning):
+    "Warned when perfoming security reducing actions"
+    pass
+
+
+class InsecureRequestWarning(SecurityWarning):
     "Warned when making an unverified HTTPS request."
     pass
+
+
+class SystemTimeWarning(SecurityWarning):
+    "Warned when system time is suspected to be wrong"
+    pass

requests/packages/urllib3/response.py

@@ -48,7 +48,10 @@ class HTTPResponse(io.IOBase):
     HTTP Response container.
 
     Backwards-compatible to httplib's HTTPResponse but the response ``body`` is
-    loaded and decoded on-demand when the ``data`` property is accessed.
+    loaded and decoded on-demand when the ``data`` property is accessed.  This
+    class is also compatible with the Python standard library's :mod:`io`
+    module, and can hence be treated as a readable object in the context of that
+    framework.
 
     Extra parameters for behaviour not present in httplib.HTTPResponse:
 
@@ -317,4 +320,14 @@ class HTTPResponse(io.IOBase):
             return self._fp.flush()
 
     def readable(self):
+        # This method is required for `io` module compatibility.
         return True
+
+    def readinto(self, b):
+        # This method is required for `io` module compatibility.
+        temp = self.read(len(b))
+        if len(temp) == 0:
+            return 0
+        else:
+            b[:len(temp)] = temp
+            return len(temp)

requests/packages/urllib3/util/response.py

@@ -5,9 +5,18 @@ def is_fp_closed(obj):
     :param obj:
         The file-like object to check.
     """
-    if hasattr(obj, 'fp'):
-        # Object is a container for another file-like object that gets released
-        # on exhaustion (e.g. HTTPResponse)
-        return obj.fp is None
 
-    return obj.closed
+    try:
+        # Check via the official file-like-object way.
+        return obj.closed
+    except AttributeError:
+        pass
+
+    try:
+        # Check if the object is a container for another file-like object that
+        # gets released on exhaustion (e.g. HTTPResponse).
+        return obj.fp is None
+    except AttributeError:
+        pass
+
+    raise ValueError("Unable to determine whether fp is closed.")

requests/packages/urllib3/util/retry.py

@@ -83,7 +83,7 @@ class Retry(object):
         same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`.
 
     :param iterable status_forcelist:
-        A set of HTTP status codes that we should force a retry on. 
+        A set of HTTP status codes that we should force a retry on.
 
         By default, this is disabled with ``None``.