diff --git a/Makefile b/Makefile index 86251957..4e3a149e 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ certs: deps: urllib3 chardet idna urllib3: - git clone https://github.com/shazow/urllib3.git && \ + git clone -b release https://github.com/shazow/urllib3.git && \ rm -fr requests/packages/urllib3 && \ cd urllib3 && \ git checkout `git describe --abbrev=0 --tags` && \ diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 6afd39b4..ac3ff838 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -32,7 +32,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.19.1' +__version__ = '1.20' __all__ = ( 'HTTPConnectionPool', diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py index e24f5e32..9f06c395 100644 --- a/requests/packages/urllib3/connection.py +++ b/requests/packages/urllib3/connection.py @@ -56,7 +56,10 @@ port_by_scheme = { 'https': 443, } -RECENT_DATE = datetime.date(2014, 1, 1) +# When updating RECENT_DATE, move it to +# within two years of the current date, and no +# earlier than 6 months ago. +RECENT_DATE = datetime.date(2016, 1, 1) class DummyConnection(object): diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 19d08f23..b4f1166a 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -25,7 +25,7 @@ from .exceptions import ( ) from .packages.ssl_match_hostname import CertificateError from .packages import six -from .packages.six.moves.queue import LifoQueue, Empty, Full +from .packages.six.moves import queue from .connection import ( port_by_scheme, DummyConnection, @@ -36,6 +36,7 @@ from .request import RequestMethods from .response import HTTPResponse from .util.connection import is_connection_dropped +from .util.request import set_file_position from .util.response import assert_header_parsing from .util.retry import Retry from .util.timeout import Timeout @@ -61,19 +62,13 @@ class ConnectionPool(object): """ scheme = None - QueueCls = LifoQueue + QueueCls = queue.LifoQueue def __init__(self, host, port=None): if not host: raise LocationValueError("No host specified.") - # httplib doesn't like it when we include brackets in ipv6 addresses - # Specifically, if we include brackets but also pass the port then - # httplib crazily doubles up the square brackets on the Host header. - # Instead, we need to make sure we never pass ``None`` as the port. - # However, for backward compatibility reasons we can't actually - # *assert* that. - self.host = host.strip('[]') + self.host = _ipv6_host(host).lower() self.port = port def __str__(self): @@ -154,7 +149,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): A dictionary with proxy headers, should not be used directly, instead, see :class:`urllib3.connectionpool.ProxyManager`" - :param \**conn_kw: + :param \\**conn_kw: Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`, :class:`urllib3.connection.HTTPSConnection` instances. """ @@ -235,7 +230,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except AttributeError: # self.pool is None raise ClosedPoolError(self, "Pool is closed.") - except Empty: + except queue.Empty: if self.block: raise EmptyPoolError(self, "Pool reached maximum size and no more " @@ -274,7 +269,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except AttributeError: # self.pool is None. pass - except Full: + except queue.Full: # This should never happen if self.block == True log.warning( "Connection pool is full, discarding connection: %s", @@ -424,7 +419,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if conn: conn.close() - except Empty: + except queue.Empty: pass # Done. def is_same_host(self, url): @@ -438,6 +433,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # TODO: Add optional support for socket.gethostbyname checking. scheme, host, port = get_host(url) + host = _ipv6_host(host).lower() + # Use explicit default port for comparison when none is given if self.port and not port: port = port_by_scheme.get(scheme) @@ -449,7 +446,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): def urlopen(self, method, url, body=None, headers=None, retries=None, redirect=True, assert_same_host=True, timeout=_Default, pool_timeout=None, release_conn=None, chunked=False, - **response_kw): + body_pos=None, **response_kw): """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all @@ -531,7 +528,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. - :param \**response_kw: + :param int body_pos: + Position to seek to in file-like body in the event of a retry or + redirect. Typically this won't need to be set because urllib3 will + auto-populate the value when needed. + + :param \\**response_kw: Additional parameters are passed to :meth:`urllib3.response.HTTPResponse.from_httplib` """ @@ -576,6 +578,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # ensures we do proper cleanup in finally. clean_exit = False + # Rewind body position, if needed. Record current position + # for future rewinds in the event of a redirect/retry. + body_pos = set_file_position(body, body_pos) + try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) @@ -612,7 +618,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Everything went great! clean_exit = True - except Empty: + except queue.Empty: # Timed out by queue. raise EmptyPoolError(self, "No pool connections are available.") @@ -668,7 +674,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): return self.urlopen(method, url, body, headers, retries, redirect, assert_same_host, timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, **response_kw) + release_conn=release_conn, body_pos=body_pos, + **response_kw) # Handle redirect? redirect_location = redirect and response.get_redirect_location() @@ -693,7 +700,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries=retries, redirect=redirect, assert_same_host=assert_same_host, timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, **response_kw) + release_conn=release_conn, body_pos=body_pos, + **response_kw) # Check if we should retry the HTTP response. has_retry_after = bool(response.getheader('Retry-After')) @@ -714,7 +722,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries=retries, redirect=redirect, assert_same_host=assert_same_host, timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, **response_kw) + release_conn=release_conn, + body_pos=body_pos, **response_kw) return response @@ -853,7 +862,7 @@ def connection_from_url(url, **kw): :param url: Absolute URL string that must include the scheme. Port is optional. - :param \**kw: + :param \\**kw: Passes additional parameters to the constructor of the appropriate :class:`.ConnectionPool`. Useful for specifying things like timeout, maxsize, headers, etc. @@ -869,3 +878,22 @@ def connection_from_url(url, **kw): return HTTPSConnectionPool(host, port=port, **kw) else: return HTTPConnectionPool(host, port=port, **kw) + + +def _ipv6_host(host): + """ + Process IPv6 address literals + """ + + # httplib doesn't like it when we include brackets in IPv6 addresses + # Specifically, if we include brackets but also pass the port then + # httplib crazily doubles up the square brackets on the Host header. + # Instead, we need to make sure we never pass ``None`` as the port. + # However, for backward compatibility reasons we can't actually + # *assert* that. See http://bugs.python.org/issue28539 + # + # Also if an IPv6 address literal has a zone identifier, the + # percent sign might be URIencoded, convert it back into ASCII + if host.startswith('[') and host.endswith(']'): + host = host.replace('%25', '%').strip('[]') + return host diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py index c6db25d4..eb4d4765 100644 --- a/requests/packages/urllib3/contrib/pyopenssl.py +++ b/requests/packages/urllib3/contrib/pyopenssl.py @@ -43,7 +43,6 @@ set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable. """ from __future__ import absolute_import -import idna import OpenSSL.SSL from cryptography import x509 from cryptography.hazmat.backends.openssl import backend as openssl_backend @@ -60,7 +59,6 @@ except ImportError: # Platform-specific: Python 3 import logging import ssl -import select import six import sys @@ -111,6 +109,8 @@ log = logging.getLogger(__name__) def inject_into_urllib3(): 'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.' + _validate_dependencies_met() + util.ssl_.SSLContext = PyOpenSSLContext util.HAS_SNI = HAS_SNI util.ssl_.HAS_SNI = HAS_SNI @@ -128,6 +128,26 @@ def extract_from_urllib3(): util.ssl_.IS_PYOPENSSL = False +def _validate_dependencies_met(): + """ + Verifies that PyOpenSSL's package-level dependencies have been met. + Throws `ImportError` if they are not met. + """ + # Method added in `cryptography==1.1`; not available in older versions + from cryptography.x509.extensions import Extensions + if getattr(Extensions, "get_extension_for_class", None) is None: + raise ImportError("'cryptography' module missing required functionality. " + "Try upgrading to v1.3.4 or newer.") + + # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509 + # attribute is only present on those versions. + from OpenSSL.crypto import X509 + x509 = X509() + if getattr(x509, "_x509", None) is None: + raise ImportError("'pyOpenSSL' module missing required functionality. " + "Try upgrading to v0.14 or newer.") + + def _dnsname_to_stdlib(name): """ Converts a dNSName SubjectAlternativeName field to the form used by the @@ -144,6 +164,8 @@ def _dnsname_to_stdlib(name): that we can't just safely call `idna.encode`: it can explode for wildcard names. This avoids that problem. """ + import idna + for prefix in [u'*.', u'.']: if name.startswith(prefix): name = name[len(prefix):] @@ -242,8 +264,7 @@ class WrappedSocket(object): else: raise except OpenSSL.SSL.WantReadError: - rd, wd, ed = select.select( - [self.socket], [], [], self.socket.gettimeout()) + rd = util.wait_for_read(self.socket, self.socket.gettimeout()) if not rd: raise timeout('The read operation timed out') else: @@ -265,8 +286,7 @@ class WrappedSocket(object): else: raise except OpenSSL.SSL.WantReadError: - rd, wd, ed = select.select( - [self.socket], [], [], self.socket.gettimeout()) + rd = util.wait_for_read(self.socket, self.socket.gettimeout()) if not rd: raise timeout('The read operation timed out') else: @@ -280,9 +300,8 @@ class WrappedSocket(object): try: return self.connection.send(data) except OpenSSL.SSL.WantWriteError: - _, wlist, _ = select.select([], [self.socket], [], - self.socket.gettimeout()) - if not wlist: + wr = util.wait_for_write(self.socket, self.socket.gettimeout()) + if not wr: raise timeout() continue @@ -416,7 +435,7 @@ class PyOpenSSLContext(object): try: cnx.do_handshake() except OpenSSL.SSL.WantReadError: - rd, _, _ = select.select([sock], [], [], sock.gettimeout()) + rd = util.wait_for_read(sock, sock.gettimeout()) if not rd: raise timeout('select timed out') continue diff --git a/requests/packages/urllib3/contrib/socks.py b/requests/packages/urllib3/contrib/socks.py index c8fa8409..39e92fde 100644 --- a/requests/packages/urllib3/contrib/socks.py +++ b/requests/packages/urllib3/contrib/socks.py @@ -83,6 +83,7 @@ class SOCKSConnection(HTTPConnection): proxy_port=self._socks_options['proxy_port'], proxy_username=self._socks_options['username'], proxy_password=self._socks_options['password'], + proxy_rdns=self._socks_options['rdns'], timeout=self.timeout, **extra_kw ) @@ -153,8 +154,16 @@ class SOCKSProxyManager(PoolManager): if parsed.scheme == 'socks5': socks_version = socks.PROXY_TYPE_SOCKS5 + rdns = False + elif parsed.scheme == 'socks5h': + socks_version = socks.PROXY_TYPE_SOCKS5 + rdns = True elif parsed.scheme == 'socks4': socks_version = socks.PROXY_TYPE_SOCKS4 + rdns = False + elif parsed.scheme == 'socks4a': + socks_version = socks.PROXY_TYPE_SOCKS4 + rdns = True else: raise ValueError( "Unable to determine SOCKS version from %s" % proxy_url @@ -168,6 +177,7 @@ class SOCKSProxyManager(PoolManager): 'proxy_port': parsed.port, 'username': username, 'password': password, + 'rdns': rdns } connection_pool_kw['_socks_options'] = socks_options diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 8a091c17..6c4be581 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -239,3 +239,8 @@ class HeaderParsingError(HTTPError): def __init__(self, defects, unparsed_data): message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data) super(HeaderParsingError, self).__init__(message) + + +class UnrewindableBodyError(HTTPError): + "urllib3 encountered an error when trying to rewind a body" + pass diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index 276b54dd..cc5a00e4 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -93,7 +93,7 @@ class PoolManager(RequestMethods): Headers to include with all requests, unless other headers are given explicitly. - :param \**connection_pool_kw: + :param \\**connection_pool_kw: Additional parameters are used to create fresh :class:`urllib3.connectionpool.ConnectionPool` instances. diff --git a/requests/packages/urllib3/util/__init__.py b/requests/packages/urllib3/util/__init__.py index 4778cf99..5ced5a44 100644 --- a/requests/packages/urllib3/util/__init__.py +++ b/requests/packages/urllib3/util/__init__.py @@ -24,6 +24,10 @@ from .url import ( split_first, Url, ) +from .wait import ( + wait_for_read, + wait_for_write +) __all__ = ( 'HAS_SNI', @@ -43,4 +47,6 @@ __all__ = ( 'resolve_ssl_version', 'split_first', 'ssl_wrap_socket', + 'wait_for_read', + 'wait_for_write' ) diff --git a/requests/packages/urllib3/util/connection.py b/requests/packages/urllib3/util/connection.py index 3d608772..bf699cfd 100644 --- a/requests/packages/urllib3/util/connection.py +++ b/requests/packages/urllib3/util/connection.py @@ -1,13 +1,7 @@ from __future__ import absolute_import import socket -try: - from select import poll, POLLIN -except ImportError: # `poll` doesn't exist on OSX and other platforms - poll = False - try: - from select import select - except ImportError: # `select` doesn't exist on AppEngine. - select = False +from .wait import wait_for_read +from .selectors import HAS_SELECT, SelectorError def is_connection_dropped(conn): # Platform-specific @@ -26,22 +20,13 @@ def is_connection_dropped(conn): # Platform-specific if sock is None: # Connection already closed (such as by httplib). return True - if not poll: - if not select: # Platform-specific: AppEngine - return False + if not HAS_SELECT: + return False - try: - return select([sock], [], [], 0.0)[0] - except socket.error: - return True - - # This version is better on platforms that support it. - p = poll() - p.register(sock, POLLIN) - for (fno, ev) in p.poll(0.0): - if fno == sock.fileno(): - # Either data is buffered (bad), or the connection is dropped. - return True + try: + return bool(wait_for_read(sock, timeout=0.0)) + except SelectorError: + return True # This function is copied from socket.py in the Python 2.7 standard diff --git a/requests/packages/urllib3/util/request.py b/requests/packages/urllib3/util/request.py index 73779315..974fc40a 100644 --- a/requests/packages/urllib3/util/request.py +++ b/requests/packages/urllib3/util/request.py @@ -1,9 +1,11 @@ from __future__ import absolute_import from base64 import b64encode -from ..packages.six import b +from ..packages.six import b, integer_types +from ..exceptions import UnrewindableBodyError ACCEPT_ENCODING = 'gzip,deflate' +_FAILEDTELL = object() def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, @@ -70,3 +72,47 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, headers['cache-control'] = 'no-cache' return headers + + +def set_file_position(body, pos): + """ + If a position is provided, move file to that point. + Otherwise, we'll attempt to record a position for future use. + """ + if pos is not None: + rewind_body(body, pos) + elif getattr(body, 'tell', None) is not None: + try: + pos = body.tell() + except (IOError, OSError): + # This differentiates from None, allowing us to catch + # a failed `tell()` later when trying to rewind the body. + pos = _FAILEDTELL + + return pos + + +def rewind_body(body, body_pos): + """ + Attempt to rewind body to a certain position. + Primarily used for request redirects and retries. + + :param body: + File-like object that supports seek. + + :param int pos: + Position to seek to in file. + """ + body_seek = getattr(body, 'seek', None) + if body_seek is not None and isinstance(body_pos, integer_types): + try: + body_seek(body_pos) + except (IOError, OSError): + raise UnrewindableBodyError("An error occured when rewinding request " + "body for redirect/retry.") + elif body_pos is _FAILEDTELL: + raise UnrewindableBodyError("Unable to record file position for rewinding " + "request body during a redirect/retry.") + else: + raise ValueError("body_pos must be of type integer, " + "instead it was %s." % type(body_pos)) diff --git a/requests/packages/urllib3/util/retry.py b/requests/packages/urllib3/util/retry.py index 47ad539a..c9e7d287 100644 --- a/requests/packages/urllib3/util/retry.py +++ b/requests/packages/urllib3/util/retry.py @@ -273,12 +273,25 @@ class Retry(object): """ return isinstance(err, (ReadTimeoutError, ProtocolError)) - def is_retry(self, method, status_code, has_retry_after=False): - """ Is this method/status code retryable? (Based on method/codes whitelists) + def _is_method_retryable(self, method): + """ Checks if a given HTTP method should be retried upon, depending if + it is included on the method whitelist. """ if self.method_whitelist and method.upper() not in self.method_whitelist: return False + return True + + def is_retry(self, method, status_code, has_retry_after=False): + """ Is this method/status code retryable? (Based on whitelists and control + variables such as the number of total retries to allow, whether to + respect the Retry-After header, whether this header is present, and + whether the returned status code is on the list of status codes to + be retried upon on the presence of the aforementioned header) + """ + if not self._is_method_retryable(method): + return False + if self.status_forcelist and status_code in self.status_forcelist: return True @@ -330,7 +343,7 @@ class Retry(object): elif error and self._is_read_error(error): # Read retry? - if read is False: + if read is False or not self._is_method_retryable(method): raise six.reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 diff --git a/requests/packages/urllib3/util/selectors.py b/requests/packages/urllib3/util/selectors.py new file mode 100644 index 00000000..51208b69 --- /dev/null +++ b/requests/packages/urllib3/util/selectors.py @@ -0,0 +1,524 @@ +# Backport of selectors.py from Python 3.5+ to support Python < 3.4 +# Also has the behavior specified in PEP 475 which is to retry syscalls +# in the case of an EINTR error. This module is required because selectors34 +# does not follow this behavior and instead returns that no dile descriptor +# events have occurred rather than retry the syscall. The decision to drop +# support for select.devpoll is made to maintain 100% test coverage. + +import errno +import math +import select +from collections import namedtuple, Mapping + +import time +try: + monotonic = time.monotonic +except (AttributeError, ImportError): # Python 3.3< + monotonic = time.time + +EVENT_READ = (1 << 0) +EVENT_WRITE = (1 << 1) + +HAS_SELECT = True # Variable that shows whether the platform has a selector. +_SYSCALL_SENTINEL = object() # Sentinel in case a system call returns None. + + +class SelectorError(Exception): + def __init__(self, errcode): + super(SelectorError, self).__init__() + self.errno = errcode + + def __repr__(self): + return "".format(self.errno) + + def __str__(self): + return self.__repr__() + + +def _fileobj_to_fd(fileobj): + """ Return a file descriptor from a file object. If + given an integer will simply return that integer back. """ + if isinstance(fileobj, int): + fd = fileobj + else: + try: + fd = int(fileobj.fileno()) + except (AttributeError, TypeError, ValueError): + raise ValueError("Invalid file object: {0!r}".format(fileobj)) + if fd < 0: + raise ValueError("Invalid file descriptor: {0}".format(fd)) + return fd + + +def _syscall_wrapper(func, recalc_timeout, *args, **kwargs): + """ Wrapper function for syscalls that could fail due to EINTR. + All functions should be retried if there is time left in the timeout + in accordance with PEP 475. """ + timeout = kwargs.get("timeout", None) + if timeout is None: + expires = None + recalc_timeout = False + else: + timeout = float(timeout) + if timeout < 0.0: # Timeout less than 0 treated as no timeout. + expires = None + else: + expires = monotonic() + timeout + + args = list(args) + if recalc_timeout and "timeout" not in kwargs: + raise ValueError( + "Timeout must be in args or kwargs to be recalculated") + + result = _SYSCALL_SENTINEL + while result is _SYSCALL_SENTINEL: + try: + result = func(*args, **kwargs) + # OSError is thrown by select.select + # IOError is thrown by select.epoll.poll + # select.error is thrown by select.poll.poll + # Aren't we thankful for Python 3.x rework for exceptions? + except (OSError, IOError, select.error) as e: + # select.error wasn't a subclass of OSError in the past. + errcode = None + if hasattr(e, "errno"): + errcode = e.errno + elif hasattr(e, "args"): + errcode = e.args[0] + + # Also test for the Windows equivalent of EINTR. + is_interrupt = (errcode == errno.EINTR or (hasattr(errno, "WSAEINTR") and + errcode == errno.WSAEINTR)) + + if is_interrupt: + if expires is not None: + current_time = monotonic() + if current_time > expires: + raise OSError(errno=errno.ETIMEDOUT) + if recalc_timeout: + if "timeout" in kwargs: + kwargs["timeout"] = expires - current_time + continue + if errcode: + raise SelectorError(errcode) + else: + raise + return result + + +SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) + + +class _SelectorMapping(Mapping): + """ Mapping of file objects to selector keys """ + + def __init__(self, selector): + self._selector = selector + + def __len__(self): + return len(self._selector._fd_to_key) + + def __getitem__(self, fileobj): + try: + fd = self._selector._fileobj_lookup(fileobj) + return self._selector._fd_to_key[fd] + except KeyError: + raise KeyError("{0!r} is not registered.".format(fileobj)) + + def __iter__(self): + return iter(self._selector._fd_to_key) + + +class BaseSelector(object): + """ Abstract Selector class + + A selector supports registering file objects to be monitored + for specific I/O events. + + A file object is a file descriptor or any object with a + `fileno()` method. An arbitrary object can be attached to the + file object which can be used for example to store context info, + a callback, etc. + + A selector can use various implementations (select(), poll(), epoll(), + and kqueue()) depending on the platform. The 'DefaultSelector' class uses + the most efficient implementation for the current platform. + """ + def __init__(self): + # Maps file descriptors to keys. + self._fd_to_key = {} + + # Read-only mapping returned by get_map() + self._map = _SelectorMapping(self) + + def _fileobj_lookup(self, fileobj): + """ Return a file descriptor from a file object. + This wraps _fileobj_to_fd() to do an exhaustive + search in case the object is invalid but we still + have it in our map. Used by unregister() so we can + unregister an object that was previously registered + even if it is closed. It is also used by _SelectorMapping + """ + try: + return _fileobj_to_fd(fileobj) + except ValueError: + + # Search through all our mapped keys. + for key in self._fd_to_key.values(): + if key.fileobj is fileobj: + return key.fd + + # Raise ValueError after all. + raise + + def register(self, fileobj, events, data=None): + """ Register a file object for a set of events to monitor. """ + if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): + raise ValueError("Invalid events: {0!r}".format(events)) + + key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) + + if key.fd in self._fd_to_key: + raise KeyError("{0!r} (FD {1}) is already registered" + .format(fileobj, key.fd)) + + self._fd_to_key[key.fd] = key + return key + + def unregister(self, fileobj): + """ Unregister a file object from being monitored. """ + try: + key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) + except KeyError: + raise KeyError("{0!r} is not registered".format(fileobj)) + return key + + def modify(self, fileobj, events, data=None): + """ Change a registered file object monitored events and data. """ + # NOTE: Some subclasses optimize this operation even further. + try: + key = self._fd_to_key[self._fileobj_lookup(fileobj)] + except KeyError: + raise KeyError("{0!r} is not registered".format(fileobj)) + + if events != key.events: + self.unregister(fileobj) + key = self.register(fileobj, events, data) + + elif data != key.data: + # Use a shortcut to update the data. + key = key._replace(data=data) + self._fd_to_key[key.fd] = key + + return key + + def select(self, timeout=None): + """ Perform the actual selection until some monitored file objects + are ready or the timeout expires. """ + raise NotImplementedError() + + def close(self): + """ Close the selector. This must be called to ensure that all + underlying resources are freed. """ + self._fd_to_key.clear() + self._map = None + + def get_key(self, fileobj): + """ Return the key associated with a registered file object. """ + mapping = self.get_map() + if mapping is None: + raise RuntimeError("Selector is closed") + try: + return mapping[fileobj] + except KeyError: + raise KeyError("{0!r} is not registered".format(fileobj)) + + def get_map(self): + """ Return a mapping of file objects to selector keys """ + return self._map + + def _key_from_fd(self, fd): + """ Return the key associated to a given file descriptor + Return None if it is not found. """ + try: + return self._fd_to_key[fd] + except KeyError: + return None + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +# Almost all platforms have select.select() +if hasattr(select, "select"): + class SelectSelector(BaseSelector): + """ Select-based selector. """ + def __init__(self): + super(SelectSelector, self).__init__() + self._readers = set() + self._writers = set() + + def register(self, fileobj, events, data=None): + key = super(SelectSelector, self).register(fileobj, events, data) + if events & EVENT_READ: + self._readers.add(key.fd) + if events & EVENT_WRITE: + self._writers.add(key.fd) + return key + + def unregister(self, fileobj): + key = super(SelectSelector, self).unregister(fileobj) + self._readers.discard(key.fd) + self._writers.discard(key.fd) + return key + + def _select(self, r, w, timeout=None): + """ Wrapper for select.select because timeout is a positional arg """ + return select.select(r, w, [], timeout) + + def select(self, timeout=None): + # Selecting on empty lists on Windows errors out. + if not len(self._readers) and not len(self._writers): + return [] + + timeout = None if timeout is None else max(timeout, 0.0) + ready = [] + r, w, _ = _syscall_wrapper(self._select, True, self._readers, + self._writers, timeout) + r = set(r) + w = set(w) + for fd in r | w: + events = 0 + if fd in r: + events |= EVENT_READ + if fd in w: + events |= EVENT_WRITE + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + +if hasattr(select, "poll"): + class PollSelector(BaseSelector): + """ Poll-based selector """ + def __init__(self): + super(PollSelector, self).__init__() + self._poll = select.poll() + + def register(self, fileobj, events, data=None): + key = super(PollSelector, self).register(fileobj, events, data) + event_mask = 0 + if events & EVENT_READ: + event_mask |= select.POLLIN + if events & EVENT_WRITE: + event_mask |= select.POLLOUT + self._poll.register(key.fd, event_mask) + return key + + def unregister(self, fileobj): + key = super(PollSelector, self).unregister(fileobj) + self._poll.unregister(key.fd) + return key + + def _wrap_poll(self, timeout=None): + """ Wrapper function for select.poll.poll() so that + _syscall_wrapper can work with only seconds. """ + if timeout is not None: + if timeout <= 0: + timeout = 0 + else: + # select.poll.poll() has a resolution of 1 millisecond, + # round away from zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) + + result = self._poll.poll(timeout) + return result + + def select(self, timeout=None): + ready = [] + fd_events = _syscall_wrapper(self._wrap_poll, True, timeout=timeout) + for fd, event_mask in fd_events: + events = 0 + if event_mask & ~select.POLLIN: + events |= EVENT_WRITE + if event_mask & ~select.POLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + + return ready + + +if hasattr(select, "epoll"): + class EpollSelector(BaseSelector): + """ Epoll-based selector """ + def __init__(self): + super(EpollSelector, self).__init__() + self._epoll = select.epoll() + + def fileno(self): + return self._epoll.fileno() + + def register(self, fileobj, events, data=None): + key = super(EpollSelector, self).register(fileobj, events, data) + events_mask = 0 + if events & EVENT_READ: + events_mask |= select.EPOLLIN + if events & EVENT_WRITE: + events_mask |= select.EPOLLOUT + _syscall_wrapper(self._epoll.register, False, key.fd, events_mask) + return key + + def unregister(self, fileobj): + key = super(EpollSelector, self).unregister(fileobj) + try: + _syscall_wrapper(self._epoll.unregister, False, key.fd) + except SelectorError: + # This can occur when the fd was closed since registry. + pass + return key + + def select(self, timeout=None): + if timeout is not None: + if timeout <= 0: + timeout = 0.0 + else: + # select.epoll.poll() has a resolution of 1 millisecond + # but luckily takes seconds so we don't need a wrapper + # like PollSelector. Just for better rounding. + timeout = math.ceil(timeout * 1e3) * 1e-3 + timeout = float(timeout) + else: + timeout = -1.0 # epoll.poll() must have a float. + + # We always want at least 1 to ensure that select can be called + # with no file descriptors registered. Otherwise will fail. + max_events = max(len(self._fd_to_key), 1) + + ready = [] + fd_events = _syscall_wrapper(self._epoll.poll, True, + timeout=timeout, + maxevents=max_events) + for fd, event_mask in fd_events: + events = 0 + if event_mask & ~select.EPOLLIN: + events |= EVENT_WRITE + if event_mask & ~select.EPOLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + self._epoll.close() + super(EpollSelector, self).close() + + +if hasattr(select, "kqueue"): + class KqueueSelector(BaseSelector): + """ Kqueue / Kevent-based selector """ + def __init__(self): + super(KqueueSelector, self).__init__() + self._kqueue = select.kqueue() + + def fileno(self): + return self._kqueue.fileno() + + def register(self, fileobj, events, data=None): + key = super(KqueueSelector, self).register(fileobj, events, data) + if events & EVENT_READ: + kevent = select.kevent(key.fd, + select.KQ_FILTER_READ, + select.KQ_EV_ADD) + + _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) + + if events & EVENT_WRITE: + kevent = select.kevent(key.fd, + select.KQ_FILTER_WRITE, + select.KQ_EV_ADD) + + _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) + + return key + + def unregister(self, fileobj): + key = super(KqueueSelector, self).unregister(fileobj) + if key.events & EVENT_READ: + kevent = select.kevent(key.fd, + select.KQ_FILTER_READ, + select.KQ_EV_DELETE) + try: + _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) + except SelectorError: + pass + if key.events & EVENT_WRITE: + kevent = select.kevent(key.fd, + select.KQ_FILTER_WRITE, + select.KQ_EV_DELETE) + try: + _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) + except SelectorError: + pass + + return key + + def select(self, timeout=None): + if timeout is not None: + timeout = max(timeout, 0) + + max_events = len(self._fd_to_key) * 2 + ready_fds = {} + + kevent_list = _syscall_wrapper(self._kqueue.control, True, + None, max_events, timeout) + + for kevent in kevent_list: + fd = kevent.ident + event_mask = kevent.filter + events = 0 + if event_mask == select.KQ_FILTER_READ: + events |= EVENT_READ + if event_mask == select.KQ_FILTER_WRITE: + events |= EVENT_WRITE + + key = self._key_from_fd(fd) + if key: + if key.fd not in ready_fds: + ready_fds[key.fd] = (key, events & key.events) + else: + old_events = ready_fds[key.fd][1] + ready_fds[key.fd] = (key, (events | old_events) & key.events) + + return list(ready_fds.values()) + + def close(self): + self._kqueue.close() + super(KqueueSelector, self).close() + + +# Choose the best implementation, roughly: +# kqueue == epoll > poll > select. Devpoll not supported. (See above) +# select() also can't accept a FD > FD_SETSIZE (usually around 1024) +if 'KqueueSelector' in globals(): # Platform-specific: Mac OS and BSD + DefaultSelector = KqueueSelector +elif 'EpollSelector' in globals(): # Platform-specific: Linux + DefaultSelector = EpollSelector +elif 'PollSelector' in globals(): # Platform-specific: Linux + DefaultSelector = PollSelector +elif 'SelectSelector' in globals(): # Platform-specific: Windows + DefaultSelector = SelectSelector +else: # Platform-specific: AppEngine + def no_selector(_): + raise ValueError("Platform does not have a selector") + DefaultSelector = no_selector + HAS_SELECT = False diff --git a/requests/packages/urllib3/util/timeout.py b/requests/packages/urllib3/util/timeout.py index 1eaa6a35..cec817e6 100644 --- a/requests/packages/urllib3/util/timeout.py +++ b/requests/packages/urllib3/util/timeout.py @@ -11,11 +11,8 @@ from ..exceptions import TimeoutStateError _Default = object() -def current_time(): - """ - Retrieve the current time. This function is mocked out in unit testing. - """ - return time.time() +# Use time.monotonic if available. +current_time = getattr(time, "monotonic", time.time) class Timeout(object): diff --git a/requests/packages/urllib3/util/wait.py b/requests/packages/urllib3/util/wait.py new file mode 100644 index 00000000..cb396e50 --- /dev/null +++ b/requests/packages/urllib3/util/wait.py @@ -0,0 +1,40 @@ +from .selectors import ( + HAS_SELECT, + DefaultSelector, + EVENT_READ, + EVENT_WRITE +) + + +def _wait_for_io_events(socks, events, timeout=None): + """ Waits for IO events to be available from a list of sockets + or optionally a single socket if passed in. Returns a list of + sockets that can be interacted with immediately. """ + if not HAS_SELECT: + raise ValueError('Platform does not have a selector') + if not isinstance(socks, list): + # Probably just a single socket. + if hasattr(socks, "fileno"): + socks = [socks] + # Otherwise it might be a non-list iterable. + else: + socks = list(socks) + with DefaultSelector() as selector: + for sock in socks: + selector.register(sock, events) + return [key[0].fileobj for key in + selector.select(timeout) if key[1] & events] + + +def wait_for_read(socks, timeout=None): + """ Waits for reading to be available from a list of sockets + or optionally a single socket if passed in. Returns a list of + sockets that can be read from immediately. """ + return _wait_for_io_events(socks, EVENT_READ, timeout) + + +def wait_for_write(socks, timeout=None): + """ Waits for writing to be available from a list of sockets + or optionally a single socket if passed in. Returns a list of + sockets that can be written to immediately. """ + return _wait_for_io_events(socks, EVENT_WRITE, timeout)