From 09bd3232dd1572f1b6c435fdcca4275687fcb937 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 5 Oct 2015 15:38:24 +0100 Subject: [PATCH] Update urllib3 to 1.12 --- requests/packages/urllib3/__init__.py | 2 +- requests/packages/urllib3/_collections.py | 58 +++++++++---------- requests/packages/urllib3/connection.py | 19 ++++-- requests/packages/urllib3/connectionpool.py | 20 ++++--- .../packages/urllib3/contrib/pyopenssl.py | 6 +- requests/packages/urllib3/exceptions.py | 3 + requests/packages/urllib3/response.py | 5 +- requests/packages/urllib3/util/connection.py | 8 +-- requests/packages/urllib3/util/response.py | 5 +- requests/packages/urllib3/util/ssl_.py | 23 +++++--- 10 files changed, 84 insertions(+), 65 deletions(-) diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index b80f19d2..86bb71d2 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = 'dev' +__version__ = '1.12' from .connectionpool import ( diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index 279416ce..b68b9a59 100644 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -97,14 +97,7 @@ class RecentlyUsedContainer(MutableMapping): return list(iterkeys(self._container)) -_dict_setitem = dict.__setitem__ -_dict_getitem = dict.__getitem__ -_dict_delitem = dict.__delitem__ -_dict_contains = dict.__contains__ -_dict_setdefault = dict.setdefault - - -class HTTPHeaderDict(dict): +class HTTPHeaderDict(MutableMapping): """ :param headers: An iterable of field-value pairs. Must not contain multiple field names @@ -139,7 +132,8 @@ class HTTPHeaderDict(dict): """ def __init__(self, headers=None, **kwargs): - dict.__init__(self) + super(HTTPHeaderDict, self).__init__() + self._container = {} if headers is not None: if isinstance(headers, HTTPHeaderDict): self._copy_from(headers) @@ -149,38 +143,44 @@ class HTTPHeaderDict(dict): self.extend(kwargs) def __setitem__(self, key, val): - return _dict_setitem(self, key.lower(), (key, val)) + self._container[key.lower()] = (key, val) + return self._container[key.lower()] def __getitem__(self, key): - val = _dict_getitem(self, key.lower()) + val = self._container[key.lower()] return ', '.join(val[1:]) def __delitem__(self, key): - return _dict_delitem(self, key.lower()) + del self._container[key.lower()] def __contains__(self, key): - return _dict_contains(self, key.lower()) + return key.lower() in self._container def __eq__(self, other): if not isinstance(other, Mapping) and not hasattr(other, 'keys'): return False if not isinstance(other, type(self)): other = type(self)(other) - return dict((k1, self[k1]) for k1 in self) == dict((k2, other[k2]) for k2 in other) + return (dict((k.lower(), v) for k, v in self.itermerged()) == + dict((k.lower(), v) for k, v in other.itermerged())) def __ne__(self, other): return not self.__eq__(other) - values = MutableMapping.values - get = MutableMapping.get - update = MutableMapping.update - if not PY3: # Python 2 iterkeys = MutableMapping.iterkeys itervalues = MutableMapping.itervalues __marker = object() + def __len__(self): + return len(self._container) + + def __iter__(self): + # Only provide the originally cased names + for vals in self._container.values(): + yield vals[0] + def pop(self, key, default=__marker): '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise KeyError is raised. @@ -216,7 +216,7 @@ class HTTPHeaderDict(dict): key_lower = key.lower() new_vals = key, val # Keep the common case aka no item present as fast as possible - vals = _dict_setdefault(self, key_lower, new_vals) + vals = self._container.setdefault(key_lower, new_vals) if new_vals is not vals: # new_vals was not inserted, as there was a previous one if isinstance(vals, list): @@ -225,7 +225,7 @@ class HTTPHeaderDict(dict): else: # vals should be a tuple then, i.e. only one item so far # Need to convert the tuple to list for further extension - _dict_setitem(self, key_lower, [vals[0], vals[1], val]) + self._container[key_lower] = [vals[0], vals[1], val] def extend(self, *args, **kwargs): """Generic import function for any type of header-like object. @@ -236,7 +236,7 @@ class HTTPHeaderDict(dict): raise TypeError("extend() takes at most 1 positional " "arguments ({} given)".format(len(args))) other = args[0] if len(args) >= 1 else () - + if isinstance(other, HTTPHeaderDict): for key, val in other.iteritems(): self.add(key, val) @@ -257,7 +257,7 @@ class HTTPHeaderDict(dict): """Returns a list of all the values for the named field. Returns an empty list if the key doesn't exist.""" try: - vals = _dict_getitem(self, key.lower()) + vals = self._container[key.lower()] except KeyError: return [] else: @@ -276,11 +276,11 @@ class HTTPHeaderDict(dict): def _copy_from(self, other): for key in other: - val = _dict_getitem(other, key) + val = other.getlist(key) if isinstance(val, list): # Don't need to convert tuples val = list(val) - _dict_setitem(self, key, val) + self._container[key.lower()] = [key] + val def copy(self): clone = type(self)() @@ -290,14 +290,14 @@ class HTTPHeaderDict(dict): def iteritems(self): """Iterate over all header lines, including duplicate ones.""" for key in self: - vals = _dict_getitem(self, key) + vals = self._container[key.lower()] for val in vals[1:]: yield vals[0], val def itermerged(self): """Iterate over all headers, merging duplicate ones together.""" for key in self: - val = _dict_getitem(self, key) + val = self._container[key.lower()] yield val[0], ', '.join(val[1:]) def items(self): @@ -307,16 +307,16 @@ class HTTPHeaderDict(dict): def from_httplib(cls, message): # Python 2 """Read headers from a Python 2 httplib message object.""" # python2.7 does not expose a proper API for exporting multiheaders - # efficiently. This function re-reads raw lines from the message + # efficiently. This function re-reads raw lines from the message # object and extracts the multiheaders properly. headers = [] - + for line in message.headers: if line.startswith((' ', '\t')): key, value = headers[-1] headers[-1] = (key, value + '\r\n' + line.rstrip()) continue - + key, value = line.split(':', 1) headers.append((key, value.strip())) diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py index f64dd1a9..3eab1e28 100644 --- a/requests/packages/urllib3/connection.py +++ b/requests/packages/urllib3/connection.py @@ -1,7 +1,7 @@ import datetime import sys import socket -from socket import timeout as SocketTimeout +from socket import error as SocketError, timeout as SocketTimeout import warnings from .packages import six @@ -36,9 +36,10 @@ except NameError: # Python 2: from .exceptions import ( + NewConnectionError, ConnectTimeoutError, - SystemTimeWarning, SubjectAltNameWarning, + SystemTimeWarning, ) from .packages.ssl_match_hostname import match_hostname @@ -133,11 +134,15 @@ class HTTPConnection(_HTTPConnection, object): conn = connection.create_connection( (self.host, self.port), self.timeout, **extra_kw) - except SocketTimeout: + except SocketTimeout as e: raise ConnectTimeoutError( self, "Connection to %s timed out. (connect timeout=%s)" % (self.host, self.timeout)) + except SocketError as e: + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e) + return conn def _prepare_conn(self, conn): @@ -185,20 +190,23 @@ class VerifiedHTTPSConnection(HTTPSConnection): """ cert_reqs = None ca_certs = None + ca_cert_dir = None ssl_version = None assert_fingerprint = None def set_cert(self, key_file=None, cert_file=None, cert_reqs=None, ca_certs=None, - assert_hostname=None, assert_fingerprint=None): + assert_hostname=None, assert_fingerprint=None, + ca_cert_dir=None): - if ca_certs and cert_reqs is None: + if (ca_certs or ca_cert_dir) and cert_reqs is None: cert_reqs = 'CERT_REQUIRED' self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs self.ca_certs = ca_certs + self.ca_cert_dir = ca_cert_dir self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint @@ -237,6 +245,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file, cert_reqs=resolved_cert_reqs, ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, server_hostname=hostname, ssl_version=resolved_ssl_version) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index c9587257..b38ac68d 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -22,10 +22,12 @@ from .exceptions import ( LocationValueError, MaxRetryError, ProxyError, + ConnectTimeoutError, ReadTimeoutError, SSLError, TimeoutError, InsecureRequestWarning, + NewConnectionError, ) from .packages.ssl_match_hostname import CertificateError from .packages import six @@ -422,7 +424,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # TODO: Add optional support for socket.gethostbyname checking. scheme, host, port = get_host(url) - + # Use explicit default port for comparison when none is given if self.port and not port: port = port_by_scheme.get(scheme) @@ -592,13 +594,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): release_conn = True raise - except (TimeoutError, HTTPException, SocketError, ConnectionError) as e: + except (TimeoutError, HTTPException, SocketError, ProtocolError) as e: # Discard the connection for these exceptions. It will be # be replaced during the next _get_conn() call. conn = conn and conn.close() release_conn = True - if isinstance(e, SocketError) and self.proxy: + if isinstance(e, (SocketError, NewConnectionError)) and self.proxy: e = ProxyError('Cannot connect to proxy.', e) elif isinstance(e, (SocketError, HTTPException)): e = ProtocolError('Connection aborted.', e) @@ -675,10 +677,10 @@ class HTTPSConnectionPool(HTTPConnectionPool): ``assert_hostname`` and ``host`` in this order to verify connections. If ``assert_hostname`` is False, no verification is done. - The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs`` and - ``ssl_version`` are only used if :mod:`ssl` is available and are fed into - :meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket - into an SSL socket. + The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, + ``ca_cert_dir``, and ``ssl_version`` are only used if :mod:`ssl` is + available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade + the connection socket into an SSL socket. """ scheme = 'https' @@ -691,7 +693,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): key_file=None, cert_file=None, cert_reqs=None, ca_certs=None, ssl_version=None, assert_hostname=None, assert_fingerprint=None, - **conn_kw): + ca_cert_dir=None, **conn_kw): HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize, block, headers, retries, _proxy, _proxy_headers, @@ -704,6 +706,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): self.cert_file = cert_file self.cert_reqs = cert_reqs self.ca_certs = ca_certs + self.ca_cert_dir = ca_cert_dir self.ssl_version = ssl_version self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint @@ -719,6 +722,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): cert_file=self.cert_file, cert_reqs=self.cert_reqs, ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, assert_hostname=self.assert_hostname, assert_fingerprint=self.assert_fingerprint) conn.ssl_version = self.ssl_version diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py index 19c5b4ee..c20ae46d 100644 --- a/requests/packages/urllib3/contrib/pyopenssl.py +++ b/requests/packages/urllib3/contrib/pyopenssl.py @@ -267,7 +267,7 @@ def _verify_callback(cnx, x509, err_no, err_depth, return_code): def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, - ssl_version=None): + ssl_version=None, ca_cert_dir=None): ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version]) if certfile: keyfile = keyfile or certfile # Match behaviour of the normal python ssl library @@ -276,9 +276,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ctx.use_privatekey_file(keyfile) if cert_reqs != ssl.CERT_NONE: ctx.set_verify(_openssl_verify[cert_reqs], _verify_callback) - if ca_certs: + if ca_certs or ca_cert_dir: try: - ctx.load_verify_locations(ca_certs, None) + ctx.load_verify_locations(ca_certs, ca_cert_dir) except OpenSSL.SSL.Error as e: raise ssl.SSLError('bad ca_certs: %r' % ca_certs, e) else: diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 36ce0d1e..9607d65f 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -112,6 +112,9 @@ class ConnectTimeoutError(TimeoutError): "Raised when a socket timeout occurs while connecting to a server" pass +class NewConnectionError(ConnectTimeoutError, PoolError): + "Raised when we fail to establish a new connection. Usually ECONNREFUSED." + pass class EmptyPoolError(PoolError): "Raised when a pool runs out of connections and no more are allowed." diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 15d4aaca..788eb6ca 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -1,7 +1,3 @@ -try: - import http.client as httplib -except ImportError: - import httplib from contextlib import contextmanager import zlib import io @@ -12,6 +8,7 @@ from .exceptions import ( ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked ) from .packages.six import string_types as basestring, binary_type, PY3 +from .packages.six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head diff --git a/requests/packages/urllib3/util/connection.py b/requests/packages/urllib3/util/connection.py index 9ed5a64c..4f2f0f18 100644 --- a/requests/packages/urllib3/util/connection.py +++ b/requests/packages/urllib3/util/connection.py @@ -80,16 +80,16 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, sock.connect(sa) return sock - except socket.error as _: - err = _ + except socket.error as e: + err = e if sock is not None: sock.close() sock = None if err is not None: raise err - else: - raise socket.error("getaddrinfo returns an empty list") + + raise socket.error("getaddrinfo returns an empty list") def _set_socket_options(sock, options): diff --git a/requests/packages/urllib3/util/response.py b/requests/packages/urllib3/util/response.py index 767ee158..2c1de154 100644 --- a/requests/packages/urllib3/util/response.py +++ b/requests/packages/urllib3/util/response.py @@ -1,7 +1,4 @@ -try: - import http.client as httplib -except ImportError: - import httplib +from ..packages.six.moves import http_client as httplib from ..exceptions import HeaderParsingError diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py index 311378bf..47b817e3 100644 --- a/requests/packages/urllib3/util/ssl_.py +++ b/requests/packages/urllib3/util/ssl_.py @@ -75,8 +75,11 @@ except ImportError: self.certfile = certfile self.keyfile = keyfile - def load_verify_locations(self, location): - self.ca_certs = location + def load_verify_locations(self, cafile=None, capath=None): + self.ca_certs = cafile + + if capath is not None: + raise SSLError("CA directories not supported in older Pythons") def set_ciphers(self, cipher_suite): if not self.supports_set_ciphers: @@ -240,10 +243,11 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, - ssl_version=None, ciphers=None, ssl_context=None): + ssl_version=None, ciphers=None, ssl_context=None, + ca_cert_dir=None): """ - All arguments except for server_hostname and ssl_context have the same - meaning as they do when using :func:`ssl.wrap_socket`. + All arguments except for server_hostname, ssl_context, and ca_cert_dir have + the same meaning as they do when using :func:`ssl.wrap_socket`. :param server_hostname: When SNI is supported, the expected hostname of the certificate @@ -253,15 +257,19 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, :param ciphers: A string of ciphers we wish the client to support. This is not supported on Python 2.6 as the ssl module does not support it. + :param ca_cert_dir: + A directory containing CA certificates in multiple separate files, as + supported by OpenSSL's -CApath flag or the capath argument to + SSLContext.load_verify_locations(). """ context = ssl_context if context is None: context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers) - if ca_certs: + if ca_certs or ca_cert_dir: try: - context.load_verify_locations(ca_certs) + context.load_verify_locations(ca_certs, ca_cert_dir) except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 raise SSLError(e) # Py33 raises FileNotFoundError which subclasses OSError @@ -270,6 +278,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if e.errno == errno.ENOENT: raise SSLError(e) raise + if certfile: context.load_cert_chain(certfile, keyfile) if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI