Update urllib3 to 1.20.0

Further, fix the Makefile to pull urllib3 from the release branch.
git-describe works based on the most recent commit but all of the
nearest commits in the graph for urllib3 are on the release branch
(since the release branch is never merged back to master).

Makefile

@@ -18,7 +18,7 @@ certs:
 deps: urllib3 chardet idna
 
 urllib3:
-	git clone https://github.com/shazow/urllib3.git && \
+	git clone -b release https://github.com/shazow/urllib3.git && \
 	    rm -fr requests/packages/urllib3 && \
 	    cd urllib3 && \
 	    git checkout `git describe --abbrev=0 --tags` && \

requests/packages/urllib3/__init__.py

@@ -32,7 +32,7 @@ except ImportError:
 
 __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
 __license__ = 'MIT'
-__version__ = '1.19.1'
+__version__ = '1.20'
 
 __all__ = (
     'HTTPConnectionPool',

requests/packages/urllib3/connection.py

@@ -56,7 +56,10 @@ port_by_scheme = {
     'https': 443,
 }
 
-RECENT_DATE = datetime.date(2014, 1, 1)
+# When updating RECENT_DATE, move it to
+# within two years of the current date, and no
+# earlier than 6 months ago.
+RECENT_DATE = datetime.date(2016, 1, 1)
 
 
 class DummyConnection(object):

requests/packages/urllib3/connectionpool.py

@@ -25,7 +25,7 @@ from .exceptions import (
 )
 from .packages.ssl_match_hostname import CertificateError
 from .packages import six
-from .packages.six.moves.queue import LifoQueue, Empty, Full
+from .packages.six.moves import queue
 from .connection import (
     port_by_scheme,
     DummyConnection,
@@ -36,6 +36,7 @@ from .request import RequestMethods
 from .response import HTTPResponse
 
 from .util.connection import is_connection_dropped
+from .util.request import set_file_position
 from .util.response import assert_header_parsing
 from .util.retry import Retry
 from .util.timeout import Timeout
@@ -61,19 +62,13 @@ class ConnectionPool(object):
     """
 
     scheme = None
-    QueueCls = LifoQueue
+    QueueCls = queue.LifoQueue
 
     def __init__(self, host, port=None):
         if not host:
             raise LocationValueError("No host specified.")
 
-        # httplib doesn't like it when we include brackets in ipv6 addresses
-        # Specifically, if we include brackets but also pass the port then
-        # httplib crazily doubles up the square brackets on the Host header.
-        # Instead, we need to make sure we never pass ``None`` as the port.
-        # However, for backward compatibility reasons we can't actually
-        # *assert* that.
-        self.host = host.strip('[]')
+        self.host = _ipv6_host(host).lower()
         self.port = port
 
     def __str__(self):
@@ -154,7 +149,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         A dictionary with proxy headers, should not be used directly,
         instead, see :class:`urllib3.connectionpool.ProxyManager`"
 
-    :param \**conn_kw:
+    :param \\**conn_kw:
         Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
         :class:`urllib3.connection.HTTPSConnection` instances.
     """
@@ -235,7 +230,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         except AttributeError:  # self.pool is None
             raise ClosedPoolError(self, "Pool is closed.")
 
-        except Empty:
+        except queue.Empty:
             if self.block:
                 raise EmptyPoolError(self,
                                      "Pool reached maximum size and no more "
@@ -274,7 +269,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         except AttributeError:
             # self.pool is None.
             pass
-        except Full:
+        except queue.Full:
             # This should never happen if self.block == True
             log.warning(
                 "Connection pool is full, discarding connection: %s",
@@ -424,7 +419,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                 if conn:
                     conn.close()
 
-        except Empty:
+        except queue.Empty:
             pass  # Done.
 
     def is_same_host(self, url):
@@ -438,6 +433,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         # TODO: Add optional support for socket.gethostbyname checking.
         scheme, host, port = get_host(url)
 
+        host = _ipv6_host(host).lower()
+
         # Use explicit default port for comparison when none is given
         if self.port and not port:
             port = port_by_scheme.get(scheme)
@@ -449,7 +446,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
     def urlopen(self, method, url, body=None, headers=None, retries=None,
                 redirect=True, assert_same_host=True, timeout=_Default,
                 pool_timeout=None, release_conn=None, chunked=False,
-                **response_kw):
+                body_pos=None, **response_kw):
         """
         Get a connection from the pool and perform an HTTP request. This is the
         lowest level call for making a request, so you'll need to specify all
@@ -531,7 +528,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             encoding. Otherwise, urllib3 will send the body using the standard
             content-length form. Defaults to False.
 
-        :param \**response_kw:
+        :param int body_pos:
+            Position to seek to in file-like body in the event of a retry or
+            redirect. Typically this won't need to be set because urllib3 will
+            auto-populate the value when needed.
+
+        :param \\**response_kw:
             Additional parameters are passed to
             :meth:`urllib3.response.HTTPResponse.from_httplib`
         """
@@ -576,6 +578,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         # ensures we do proper cleanup in finally.
         clean_exit = False
 
+        # Rewind body position, if needed. Record current position
+        # for future rewinds in the event of a redirect/retry.
+        body_pos = set_file_position(body, body_pos)
+
         try:
             # Request a connection from the queue.
             timeout_obj = self._get_timeout(timeout)
@@ -612,7 +618,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # Everything went great!
             clean_exit = True
 
-        except Empty:
+        except queue.Empty:
             # Timed out by queue.
             raise EmptyPoolError(self, "No pool connections are available.")
 
@@ -668,7 +674,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             return self.urlopen(method, url, body, headers, retries,
                                 redirect, assert_same_host,
                                 timeout=timeout, pool_timeout=pool_timeout,
-                                release_conn=release_conn, **response_kw)
+                                release_conn=release_conn, body_pos=body_pos,
+                                **response_kw)
 
         # Handle redirect?
         redirect_location = redirect and response.get_redirect_location()
@@ -693,7 +700,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                 retries=retries, redirect=redirect,
                 assert_same_host=assert_same_host,
                 timeout=timeout, pool_timeout=pool_timeout,
-                release_conn=release_conn, **response_kw)
+                release_conn=release_conn, body_pos=body_pos,
+                **response_kw)
 
         # Check if we should retry the HTTP response.
         has_retry_after = bool(response.getheader('Retry-After'))
@@ -714,7 +722,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                 retries=retries, redirect=redirect,
                 assert_same_host=assert_same_host,
                 timeout=timeout, pool_timeout=pool_timeout,
-                release_conn=release_conn, **response_kw)
+                release_conn=release_conn,
+                body_pos=body_pos, **response_kw)
 
         return response
 
@@ -853,7 +862,7 @@ def connection_from_url(url, **kw):
     :param url:
         Absolute URL string that must include the scheme. Port is optional.
 
-    :param \**kw:
+    :param \\**kw:
         Passes additional parameters to the constructor of the appropriate
         :class:`.ConnectionPool`. Useful for specifying things like
         timeout, maxsize, headers, etc.
@@ -869,3 +878,22 @@ def connection_from_url(url, **kw):
         return HTTPSConnectionPool(host, port=port, **kw)
     else:
         return HTTPConnectionPool(host, port=port, **kw)
+
+
+def _ipv6_host(host):
+    """
+    Process IPv6 address literals
+    """
+
+    # httplib doesn't like it when we include brackets in IPv6 addresses
+    # Specifically, if we include brackets but also pass the port then
+    # httplib crazily doubles up the square brackets on the Host header.
+    # Instead, we need to make sure we never pass ``None`` as the port.
+    # However, for backward compatibility reasons we can't actually
+    # *assert* that.  See http://bugs.python.org/issue28539
+    #
+    # Also if an IPv6 address literal has a zone identifier, the
+    # percent sign might be URIencoded, convert it back into ASCII
+    if host.startswith('[') and host.endswith(']'):
+        host = host.replace('%25', '%').strip('[]')
+    return host

requests/packages/urllib3/contrib/pyopenssl.py

@@ -43,7 +43,6 @@ set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.
 """
 from __future__ import absolute_import
 
-import idna
 import OpenSSL.SSL
 from cryptography import x509
 from cryptography.hazmat.backends.openssl import backend as openssl_backend
@@ -60,7 +59,6 @@ except ImportError:  # Platform-specific: Python 3
 
 import logging
 import ssl
-import select
 import six
 import sys
 
@@ -111,6 +109,8 @@ log = logging.getLogger(__name__)
 def inject_into_urllib3():
     'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.'
 
+    _validate_dependencies_met()
+
     util.ssl_.SSLContext = PyOpenSSLContext
     util.HAS_SNI = HAS_SNI
     util.ssl_.HAS_SNI = HAS_SNI
@@ -128,6 +128,26 @@ def extract_from_urllib3():
     util.ssl_.IS_PYOPENSSL = False
 
 
+def _validate_dependencies_met():
+    """
+    Verifies that PyOpenSSL's package-level dependencies have been met.
+    Throws `ImportError` if they are not met.
+    """
+    # Method added in `cryptography==1.1`; not available in older versions
+    from cryptography.x509.extensions import Extensions
+    if getattr(Extensions, "get_extension_for_class", None) is None:
+        raise ImportError("'cryptography' module missing required functionality.  "
+                          "Try upgrading to v1.3.4 or newer.")
+
+    # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509
+    # attribute is only present on those versions.
+    from OpenSSL.crypto import X509
+    x509 = X509()
+    if getattr(x509, "_x509", None) is None:
+        raise ImportError("'pyOpenSSL' module missing required functionality. "
+                          "Try upgrading to v0.14 or newer.")
+
+
 def _dnsname_to_stdlib(name):
     """
     Converts a dNSName SubjectAlternativeName field to the form used by the
@@ -144,6 +164,8 @@ def _dnsname_to_stdlib(name):
         that we can't just safely call `idna.encode`: it can explode for
         wildcard names. This avoids that problem.
         """
+        import idna
+
         for prefix in [u'*.', u'.']:
             if name.startswith(prefix):
                 name = name[len(prefix):]
@@ -242,8 +264,7 @@ class WrappedSocket(object):
             else:
                 raise
         except OpenSSL.SSL.WantReadError:
-            rd, wd, ed = select.select(
-                [self.socket], [], [], self.socket.gettimeout())
+            rd = util.wait_for_read(self.socket, self.socket.gettimeout())
             if not rd:
                 raise timeout('The read operation timed out')
             else:
@@ -265,8 +286,7 @@ class WrappedSocket(object):
             else:
                 raise
         except OpenSSL.SSL.WantReadError:
-            rd, wd, ed = select.select(
-                [self.socket], [], [], self.socket.gettimeout())
+            rd = util.wait_for_read(self.socket, self.socket.gettimeout())
             if not rd:
                 raise timeout('The read operation timed out')
             else:
@@ -280,9 +300,8 @@ class WrappedSocket(object):
             try:
                 return self.connection.send(data)
             except OpenSSL.SSL.WantWriteError:
-                _, wlist, _ = select.select([], [self.socket], [],
-                                            self.socket.gettimeout())
-                if not wlist:
+                wr = util.wait_for_write(self.socket, self.socket.gettimeout())
+                if not wr:
                     raise timeout()
                 continue
 
@@ -416,7 +435,7 @@ class PyOpenSSLContext(object):
             try:
                 cnx.do_handshake()
             except OpenSSL.SSL.WantReadError:
-                rd, _, _ = select.select([sock], [], [], sock.gettimeout())
+                rd = util.wait_for_read(sock, sock.gettimeout())
                 if not rd:
                     raise timeout('select timed out')
                 continue

requests/packages/urllib3/contrib/socks.py

@@ -83,6 +83,7 @@ class SOCKSConnection(HTTPConnection):
                 proxy_port=self._socks_options['proxy_port'],
                 proxy_username=self._socks_options['username'],
                 proxy_password=self._socks_options['password'],
+                proxy_rdns=self._socks_options['rdns'],
                 timeout=self.timeout,
                 **extra_kw
             )
@@ -153,8 +154,16 @@ class SOCKSProxyManager(PoolManager):
 
         if parsed.scheme == 'socks5':
             socks_version = socks.PROXY_TYPE_SOCKS5
+            rdns = False
+        elif parsed.scheme == 'socks5h':
+            socks_version = socks.PROXY_TYPE_SOCKS5
+            rdns = True
         elif parsed.scheme == 'socks4':
             socks_version = socks.PROXY_TYPE_SOCKS4
+            rdns = False
+        elif parsed.scheme == 'socks4a':
+            socks_version = socks.PROXY_TYPE_SOCKS4
+            rdns = True
         else:
             raise ValueError(
                 "Unable to determine SOCKS version from %s" % proxy_url
@@ -168,6 +177,7 @@ class SOCKSProxyManager(PoolManager):
             'proxy_port': parsed.port,
             'username': username,
             'password': password,
+            'rdns': rdns
         }
         connection_pool_kw['_socks_options'] = socks_options
 

requests/packages/urllib3/exceptions.py

@@ -239,3 +239,8 @@ class HeaderParsingError(HTTPError):
     def __init__(self, defects, unparsed_data):
         message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data)
         super(HeaderParsingError, self).__init__(message)
+
+
+class UnrewindableBodyError(HTTPError):
+    "urllib3 encountered an error when trying to rewind a body"
+    pass

requests/packages/urllib3/poolmanager.py

@@ -93,7 +93,7 @@ class PoolManager(RequestMethods):
         Headers to include with all requests, unless other headers are given
         explicitly.
 
-    :param \**connection_pool_kw:
+    :param \\**connection_pool_kw:
         Additional parameters are used to create fresh
         :class:`urllib3.connectionpool.ConnectionPool` instances.
 

requests/packages/urllib3/util/__init__.py

@@ -24,6 +24,10 @@ from .url import (
     split_first,
     Url,
 )
+from .wait import (
+    wait_for_read,
+    wait_for_write
+)
 
 __all__ = (
     'HAS_SNI',
@@ -43,4 +47,6 @@ __all__ = (
     'resolve_ssl_version',
     'split_first',
     'ssl_wrap_socket',
+    'wait_for_read',
+    'wait_for_write'
 )

requests/packages/urllib3/util/connection.py

@@ -1,13 +1,7 @@
 from __future__ import absolute_import
 import socket
-try:
-    from select import poll, POLLIN
-except ImportError:  # `poll` doesn't exist on OSX and other platforms
-    poll = False
-    try:
-        from select import select
-    except ImportError:  # `select` doesn't exist on AppEngine.
-        select = False
+from .wait import wait_for_read
+from .selectors import HAS_SELECT, SelectorError
 
 
 def is_connection_dropped(conn):  # Platform-specific
@@ -26,22 +20,13 @@ def is_connection_dropped(conn):  # Platform-specific
     if sock is None:  # Connection already closed (such as by httplib).
         return True
 
-    if not poll:
-        if not select:  # Platform-specific: AppEngine
-            return False
+    if not HAS_SELECT:
+        return False
 
-        try:
-            return select([sock], [], [], 0.0)[0]
-        except socket.error:
-            return True
-
-    # This version is better on platforms that support it.
-    p = poll()
-    p.register(sock, POLLIN)
-    for (fno, ev) in p.poll(0.0):
-        if fno == sock.fileno():
-            # Either data is buffered (bad), or the connection is dropped.
-            return True
+    try:
+        return bool(wait_for_read(sock, timeout=0.0))
+    except SelectorError:
+        return True
 
 
 # This function is copied from socket.py in the Python 2.7 standard

requests/packages/urllib3/util/request.py

@@ -1,9 +1,11 @@
 from __future__ import absolute_import
 from base64 import b64encode
 
-from ..packages.six import b
+from ..packages.six import b, integer_types
+from ..exceptions import UnrewindableBodyError
 
 ACCEPT_ENCODING = 'gzip,deflate'
+_FAILEDTELL = object()
 
 
 def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
@@ -70,3 +72,47 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
         headers['cache-control'] = 'no-cache'
 
     return headers
+
+
+def set_file_position(body, pos):
+    """
+    If a position is provided, move file to that point.
+    Otherwise, we'll attempt to record a position for future use.
+    """
+    if pos is not None:
+        rewind_body(body, pos)
+    elif getattr(body, 'tell', None) is not None:
+        try:
+            pos = body.tell()
+        except (IOError, OSError):
+            # This differentiates from None, allowing us to catch
+            # a failed `tell()` later when trying to rewind the body.
+            pos = _FAILEDTELL
+
+    return pos
+
+
+def rewind_body(body, body_pos):
+    """
+    Attempt to rewind body to a certain position.
+    Primarily used for request redirects and retries.
+
+    :param body:
+        File-like object that supports seek.
+
+    :param int pos:
+        Position to seek to in file.
+    """
+    body_seek = getattr(body, 'seek', None)
+    if body_seek is not None and isinstance(body_pos, integer_types):
+        try:
+            body_seek(body_pos)
+        except (IOError, OSError):
+            raise UnrewindableBodyError("An error occured when rewinding request "
+                                        "body for redirect/retry.")
+    elif body_pos is _FAILEDTELL:
+        raise UnrewindableBodyError("Unable to record file position for rewinding "
+                                    "request body during a redirect/retry.")
+    else:
+        raise ValueError("body_pos must be of type integer, "
+                         "instead it was %s." % type(body_pos))

requests/packages/urllib3/util/retry.py

@@ -273,12 +273,25 @@ class Retry(object):
         """
         return isinstance(err, (ReadTimeoutError, ProtocolError))
 
-    def is_retry(self, method, status_code, has_retry_after=False):
-        """ Is this method/status code retryable? (Based on method/codes whitelists)
+    def _is_method_retryable(self, method):
+        """ Checks if a given HTTP method should be retried upon, depending if
+        it is included on the method whitelist.
         """
         if self.method_whitelist and method.upper() not in self.method_whitelist:
             return False
 
+        return True
+
+    def is_retry(self, method, status_code, has_retry_after=False):
+        """ Is this method/status code retryable? (Based on whitelists and control
+        variables such as the number of total retries to allow, whether to
+        respect the Retry-After header, whether this header is present, and
+        whether the returned status code is on the list of status codes to
+        be retried upon on the presence of the aforementioned header)
+        """
+        if not self._is_method_retryable(method):
+            return False
+
         if self.status_forcelist and status_code in self.status_forcelist:
             return True
 
@@ -330,7 +343,7 @@ class Retry(object):
 
         elif error and self._is_read_error(error):
             # Read retry?
-            if read is False:
+            if read is False or not self._is_method_retryable(method):
                 raise six.reraise(type(error), error, _stacktrace)
             elif read is not None:
                 read -= 1

requests/packages/urllib3/util/selectors.py

@@ -0,0 +1,524 @@
+# Backport of selectors.py from Python 3.5+ to support Python < 3.4
+# Also has the behavior specified in PEP 475 which is to retry syscalls
+# in the case of an EINTR error. This module is required because selectors34
+# does not follow this behavior and instead returns that no dile descriptor
+# events have occurred rather than retry the syscall. The decision to drop
+# support for select.devpoll is made to maintain 100% test coverage.
+
+import errno
+import math
+import select
+from collections import namedtuple, Mapping
+
+import time
+try:
+    monotonic = time.monotonic
+except (AttributeError, ImportError):  # Python 3.3<
+    monotonic = time.time
+
+EVENT_READ = (1 << 0)
+EVENT_WRITE = (1 << 1)
+
+HAS_SELECT = True  # Variable that shows whether the platform has a selector.
+_SYSCALL_SENTINEL = object()  # Sentinel in case a system call returns None.
+
+
+class SelectorError(Exception):
+    def __init__(self, errcode):
+        super(SelectorError, self).__init__()
+        self.errno = errcode
+
+    def __repr__(self):
+        return "<SelectorError errno={0}>".format(self.errno)
+
+    def __str__(self):
+        return self.__repr__()
+
+
+def _fileobj_to_fd(fileobj):
+    """ Return a file descriptor from a file object. If
+    given an integer will simply return that integer back. """
+    if isinstance(fileobj, int):
+        fd = fileobj
+    else:
+        try:
+            fd = int(fileobj.fileno())
+        except (AttributeError, TypeError, ValueError):
+            raise ValueError("Invalid file object: {0!r}".format(fileobj))
+    if fd < 0:
+        raise ValueError("Invalid file descriptor: {0}".format(fd))
+    return fd
+
+
+def _syscall_wrapper(func, recalc_timeout, *args, **kwargs):
+    """ Wrapper function for syscalls that could fail due to EINTR.
+    All functions should be retried if there is time left in the timeout
+    in accordance with PEP 475. """
+    timeout = kwargs.get("timeout", None)
+    if timeout is None:
+        expires = None
+        recalc_timeout = False
+    else:
+        timeout = float(timeout)
+        if timeout < 0.0:  # Timeout less than 0 treated as no timeout.
+            expires = None
+        else:
+            expires = monotonic() + timeout
+
+    args = list(args)
+    if recalc_timeout and "timeout" not in kwargs:
+        raise ValueError(
+            "Timeout must be in args or kwargs to be recalculated")
+
+    result = _SYSCALL_SENTINEL
+    while result is _SYSCALL_SENTINEL:
+        try:
+            result = func(*args, **kwargs)
+        # OSError is thrown by select.select
+        # IOError is thrown by select.epoll.poll
+        # select.error is thrown by select.poll.poll
+        # Aren't we thankful for Python 3.x rework for exceptions?
+        except (OSError, IOError, select.error) as e:
+            # select.error wasn't a subclass of OSError in the past.
+            errcode = None
+            if hasattr(e, "errno"):
+                errcode = e.errno
+            elif hasattr(e, "args"):
+                errcode = e.args[0]
+
+            # Also test for the Windows equivalent of EINTR.
+            is_interrupt = (errcode == errno.EINTR or (hasattr(errno, "WSAEINTR") and
+                                                       errcode == errno.WSAEINTR))
+
+            if is_interrupt:
+                if expires is not None:
+                    current_time = monotonic()
+                    if current_time > expires:
+                        raise OSError(errno=errno.ETIMEDOUT)
+                    if recalc_timeout:
+                        if "timeout" in kwargs:
+                            kwargs["timeout"] = expires - current_time
+                continue
+            if errcode:
+                raise SelectorError(errcode)
+            else:
+                raise
+    return result
+
+
+SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data'])
+
+
+class _SelectorMapping(Mapping):
+    """ Mapping of file objects to selector keys """
+
+    def __init__(self, selector):
+        self._selector = selector
+
+    def __len__(self):
+        return len(self._selector._fd_to_key)
+
+    def __getitem__(self, fileobj):
+        try:
+            fd = self._selector._fileobj_lookup(fileobj)
+            return self._selector._fd_to_key[fd]
+        except KeyError:
+            raise KeyError("{0!r} is not registered.".format(fileobj))
+
+    def __iter__(self):
+        return iter(self._selector._fd_to_key)
+
+
+class BaseSelector(object):
+    """ Abstract Selector class
+
+    A selector supports registering file objects to be monitored
+    for specific I/O events.
+
+    A file object is a file descriptor or any object with a
+    `fileno()` method. An arbitrary object can be attached to the
+    file object which can be used for example to store context info,
+    a callback, etc.
+
+    A selector can use various implementations (select(), poll(), epoll(),
+    and kqueue()) depending on the platform. The 'DefaultSelector' class uses
+    the most efficient implementation for the current platform.
+    """
+    def __init__(self):
+        # Maps file descriptors to keys.
+        self._fd_to_key = {}
+
+        # Read-only mapping returned by get_map()
+        self._map = _SelectorMapping(self)
+
+    def _fileobj_lookup(self, fileobj):
+        """ Return a file descriptor from a file object.
+        This wraps _fileobj_to_fd() to do an exhaustive
+        search in case the object is invalid but we still
+        have it in our map. Used by unregister() so we can
+        unregister an object that was previously registered
+        even if it is closed. It is also used by _SelectorMapping
+        """
+        try:
+            return _fileobj_to_fd(fileobj)
+        except ValueError:
+
+            # Search through all our mapped keys.
+            for key in self._fd_to_key.values():
+                if key.fileobj is fileobj:
+                    return key.fd
+
+            # Raise ValueError after all.
+            raise
+
+    def register(self, fileobj, events, data=None):
+        """ Register a file object for a set of events to monitor. """
+        if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)):
+            raise ValueError("Invalid events: {0!r}".format(events))
+
+        key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data)
+
+        if key.fd in self._fd_to_key:
+            raise KeyError("{0!r} (FD {1}) is already registered"
+                           .format(fileobj, key.fd))
+
+        self._fd_to_key[key.fd] = key
+        return key
+
+    def unregister(self, fileobj):
+        """ Unregister a file object from being monitored. """
+        try:
+            key = self._fd_to_key.pop(self._fileobj_lookup(fileobj))
+        except KeyError:
+            raise KeyError("{0!r} is not registered".format(fileobj))
+        return key
+
+    def modify(self, fileobj, events, data=None):
+        """ Change a registered file object monitored events and data. """
+        # NOTE: Some subclasses optimize this operation even further.
+        try:
+            key = self._fd_to_key[self._fileobj_lookup(fileobj)]
+        except KeyError:
+            raise KeyError("{0!r} is not registered".format(fileobj))
+
+        if events != key.events:
+            self.unregister(fileobj)
+            key = self.register(fileobj, events, data)
+
+        elif data != key.data:
+            # Use a shortcut to update the data.
+            key = key._replace(data=data)
+            self._fd_to_key[key.fd] = key
+
+        return key
+
+    def select(self, timeout=None):
+        """ Perform the actual selection until some monitored file objects
+        are ready or the timeout expires. """
+        raise NotImplementedError()
+
+    def close(self):
+        """ Close the selector. This must be called to ensure that all
+        underlying resources are freed. """
+        self._fd_to_key.clear()
+        self._map = None
+
+    def get_key(self, fileobj):
+        """ Return the key associated with a registered file object. """
+        mapping = self.get_map()
+        if mapping is None:
+            raise RuntimeError("Selector is closed")
+        try:
+            return mapping[fileobj]
+        except KeyError:
+            raise KeyError("{0!r} is not registered".format(fileobj))
+
+    def get_map(self):
+        """ Return a mapping of file objects to selector keys """
+        return self._map
+
+    def _key_from_fd(self, fd):
+        """ Return the key associated to a given file descriptor
+         Return None if it is not found. """
+        try:
+            return self._fd_to_key[fd]
+        except KeyError:
+            return None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+
+# Almost all platforms have select.select()
+if hasattr(select, "select"):
+    class SelectSelector(BaseSelector):
+        """ Select-based selector. """
+        def __init__(self):
+            super(SelectSelector, self).__init__()
+            self._readers = set()
+            self._writers = set()
+
+        def register(self, fileobj, events, data=None):
+            key = super(SelectSelector, self).register(fileobj, events, data)
+            if events & EVENT_READ:
+                self._readers.add(key.fd)
+            if events & EVENT_WRITE:
+                self._writers.add(key.fd)
+            return key
+
+        def unregister(self, fileobj):
+            key = super(SelectSelector, self).unregister(fileobj)
+            self._readers.discard(key.fd)
+            self._writers.discard(key.fd)
+            return key
+
+        def _select(self, r, w, timeout=None):
+            """ Wrapper for select.select because timeout is a positional arg """
+            return select.select(r, w, [], timeout)
+
+        def select(self, timeout=None):
+            # Selecting on empty lists on Windows errors out.
+            if not len(self._readers) and not len(self._writers):
+                return []
+
+            timeout = None if timeout is None else max(timeout, 0.0)
+            ready = []
+            r, w, _ = _syscall_wrapper(self._select, True, self._readers,
+                                       self._writers, timeout)
+            r = set(r)
+            w = set(w)
+            for fd in r | w:
+                events = 0
+                if fd in r:
+                    events |= EVENT_READ
+                if fd in w:
+                    events |= EVENT_WRITE
+
+                key = self._key_from_fd(fd)
+                if key:
+                    ready.append((key, events & key.events))
+            return ready
+
+
+if hasattr(select, "poll"):
+    class PollSelector(BaseSelector):
+        """ Poll-based selector """
+        def __init__(self):
+            super(PollSelector, self).__init__()
+            self._poll = select.poll()
+
+        def register(self, fileobj, events, data=None):
+            key = super(PollSelector, self).register(fileobj, events, data)
+            event_mask = 0
+            if events & EVENT_READ:
+                event_mask |= select.POLLIN
+            if events & EVENT_WRITE:
+                event_mask |= select.POLLOUT
+            self._poll.register(key.fd, event_mask)
+            return key
+
+        def unregister(self, fileobj):
+            key = super(PollSelector, self).unregister(fileobj)
+            self._poll.unregister(key.fd)
+            return key
+
+        def _wrap_poll(self, timeout=None):
+            """ Wrapper function for select.poll.poll() so that
+            _syscall_wrapper can work with only seconds. """
+            if timeout is not None:
+                if timeout <= 0:
+                    timeout = 0
+                else:
+                    # select.poll.poll() has a resolution of 1 millisecond,
+                    # round away from zero to wait *at least* timeout seconds.
+                    timeout = math.ceil(timeout * 1e3)
+
+            result = self._poll.poll(timeout)
+            return result
+
+        def select(self, timeout=None):
+            ready = []
+            fd_events = _syscall_wrapper(self._wrap_poll, True, timeout=timeout)
+            for fd, event_mask in fd_events:
+                events = 0
+                if event_mask & ~select.POLLIN:
+                    events |= EVENT_WRITE
+                if event_mask & ~select.POLLOUT:
+                    events |= EVENT_READ
+
+                key = self._key_from_fd(fd)
+                if key:
+                    ready.append((key, events & key.events))
+
+            return ready
+
+
+if hasattr(select, "epoll"):
+    class EpollSelector(BaseSelector):
+        """ Epoll-based selector """
+        def __init__(self):
+            super(EpollSelector, self).__init__()
+            self._epoll = select.epoll()
+
+        def fileno(self):
+            return self._epoll.fileno()
+
+        def register(self, fileobj, events, data=None):
+            key = super(EpollSelector, self).register(fileobj, events, data)
+            events_mask = 0
+            if events & EVENT_READ:
+                events_mask |= select.EPOLLIN
+            if events & EVENT_WRITE:
+                events_mask |= select.EPOLLOUT
+            _syscall_wrapper(self._epoll.register, False, key.fd, events_mask)
+            return key
+
+        def unregister(self, fileobj):
+            key = super(EpollSelector, self).unregister(fileobj)
+            try:
+                _syscall_wrapper(self._epoll.unregister, False, key.fd)
+            except SelectorError:
+                # This can occur when the fd was closed since registry.
+                pass
+            return key
+
+        def select(self, timeout=None):
+            if timeout is not None:
+                if timeout <= 0:
+                    timeout = 0.0
+                else:
+                    # select.epoll.poll() has a resolution of 1 millisecond
+                    # but luckily takes seconds so we don't need a wrapper
+                    # like PollSelector. Just for better rounding.
+                    timeout = math.ceil(timeout * 1e3) * 1e-3
+                timeout = float(timeout)
+            else:
+                timeout = -1.0  # epoll.poll() must have a float.
+
+            # We always want at least 1 to ensure that select can be called
+            # with no file descriptors registered. Otherwise will fail.
+            max_events = max(len(self._fd_to_key), 1)
+
+            ready = []
+            fd_events = _syscall_wrapper(self._epoll.poll, True,
+                                         timeout=timeout,
+                                         maxevents=max_events)
+            for fd, event_mask in fd_events:
+                events = 0
+                if event_mask & ~select.EPOLLIN:
+                    events |= EVENT_WRITE
+                if event_mask & ~select.EPOLLOUT:
+                    events |= EVENT_READ
+
+                key = self._key_from_fd(fd)
+                if key:
+                    ready.append((key, events & key.events))
+            return ready
+
+        def close(self):
+            self._epoll.close()
+            super(EpollSelector, self).close()
+
+
+if hasattr(select, "kqueue"):
+    class KqueueSelector(BaseSelector):
+        """ Kqueue / Kevent-based selector """
+        def __init__(self):
+            super(KqueueSelector, self).__init__()
+            self._kqueue = select.kqueue()
+
+        def fileno(self):
+            return self._kqueue.fileno()
+
+        def register(self, fileobj, events, data=None):
+            key = super(KqueueSelector, self).register(fileobj, events, data)
+            if events & EVENT_READ:
+                kevent = select.kevent(key.fd,
+                                       select.KQ_FILTER_READ,
+                                       select.KQ_EV_ADD)
+
+                _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+
+            if events & EVENT_WRITE:
+                kevent = select.kevent(key.fd,
+                                       select.KQ_FILTER_WRITE,
+                                       select.KQ_EV_ADD)
+
+                _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+
+            return key
+
+        def unregister(self, fileobj):
+            key = super(KqueueSelector, self).unregister(fileobj)
+            if key.events & EVENT_READ:
+                kevent = select.kevent(key.fd,
+                                       select.KQ_FILTER_READ,
+                                       select.KQ_EV_DELETE)
+                try:
+                    _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+                except SelectorError:
+                    pass
+            if key.events & EVENT_WRITE:
+                kevent = select.kevent(key.fd,
+                                       select.KQ_FILTER_WRITE,
+                                       select.KQ_EV_DELETE)
+                try:
+                    _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+                except SelectorError:
+                    pass
+
+            return key
+
+        def select(self, timeout=None):
+            if timeout is not None:
+                timeout = max(timeout, 0)
+
+            max_events = len(self._fd_to_key) * 2
+            ready_fds = {}
+
+            kevent_list = _syscall_wrapper(self._kqueue.control, True,
+                                           None, max_events, timeout)
+
+            for kevent in kevent_list:
+                fd = kevent.ident
+                event_mask = kevent.filter
+                events = 0
+                if event_mask == select.KQ_FILTER_READ:
+                    events |= EVENT_READ
+                if event_mask == select.KQ_FILTER_WRITE:
+                    events |= EVENT_WRITE
+
+                key = self._key_from_fd(fd)
+                if key:
+                    if key.fd not in ready_fds:
+                        ready_fds[key.fd] = (key, events & key.events)
+                    else:
+                        old_events = ready_fds[key.fd][1]
+                        ready_fds[key.fd] = (key, (events | old_events) & key.events)
+
+            return list(ready_fds.values())
+
+        def close(self):
+            self._kqueue.close()
+            super(KqueueSelector, self).close()
+
+
+# Choose the best implementation, roughly:
+# kqueue == epoll > poll > select. Devpoll not supported. (See above)
+# select() also can't accept a FD > FD_SETSIZE (usually around 1024)
+if 'KqueueSelector' in globals():  # Platform-specific: Mac OS and BSD
+    DefaultSelector = KqueueSelector
+elif 'EpollSelector' in globals():  # Platform-specific: Linux
+    DefaultSelector = EpollSelector
+elif 'PollSelector' in globals():  # Platform-specific: Linux
+    DefaultSelector = PollSelector
+elif 'SelectSelector' in globals():  # Platform-specific: Windows
+    DefaultSelector = SelectSelector
+else:  # Platform-specific: AppEngine
+    def no_selector(_):
+        raise ValueError("Platform does not have a selector")
+    DefaultSelector = no_selector
+    HAS_SELECT = False

requests/packages/urllib3/util/timeout.py

@@ -11,11 +11,8 @@ from ..exceptions import TimeoutStateError
 _Default = object()
 
 
-def current_time():
-    """
-    Retrieve the current time. This function is mocked out in unit testing.
-    """
-    return time.time()
+# Use time.monotonic if available.
+current_time = getattr(time, "monotonic", time.time)
 
 
 class Timeout(object):

requests/packages/urllib3/util/wait.py

@@ -0,0 +1,40 @@
+from .selectors import (
+    HAS_SELECT,
+    DefaultSelector,
+    EVENT_READ,
+    EVENT_WRITE
+)
+
+
+def _wait_for_io_events(socks, events, timeout=None):
+    """ Waits for IO events to be available from a list of sockets
+    or optionally a single socket if passed in. Returns a list of
+    sockets that can be interacted with immediately. """
+    if not HAS_SELECT:
+        raise ValueError('Platform does not have a selector')
+    if not isinstance(socks, list):
+        # Probably just a single socket.
+        if hasattr(socks, "fileno"):
+            socks = [socks]
+        # Otherwise it might be a non-list iterable.
+        else:
+            socks = list(socks)
+    with DefaultSelector() as selector:
+        for sock in socks:
+            selector.register(sock, events)
+        return [key[0].fileobj for key in
+                selector.select(timeout) if key[1] & events]
+
+
+def wait_for_read(socks, timeout=None):
+    """ Waits for reading to be available from a list of sockets
+    or optionally a single socket if passed in. Returns a list of
+    sockets that can be read from immediately. """
+    return _wait_for_io_events(socks, EVENT_READ, timeout)
+
+
+def wait_for_write(socks, timeout=None):
+    """ Waits for writing to be available from a list of sockets
+    or optionally a single socket if passed in. Returns a list of
+    sockets that can be written to immediately. """
+    return _wait_for_io_events(socks, EVENT_WRITE, timeout)