diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index c3536742..4aa596d0 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -32,7 +32,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.16' +__version__ = '1.19' __all__ = ( 'HTTPConnectionPool', diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py index 5ce00804..e24f5e32 100644 --- a/requests/packages/urllib3/connection.py +++ b/requests/packages/urllib3/connection.py @@ -7,13 +7,8 @@ import socket from socket import error as SocketError, timeout as SocketTimeout import warnings from .packages import six - -try: # Python 3 - from http.client import HTTPConnection as _HTTPConnection - from http.client import HTTPException # noqa: unused in this module -except ImportError: - from httplib import HTTPConnection as _HTTPConnection - from httplib import HTTPException # noqa: unused in this module +from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection +from .packages.six.moves.http_client import HTTPException # noqa: F401 try: # Compiled with SSL? import ssl @@ -44,8 +39,9 @@ from .packages.ssl_match_hostname import match_hostname, CertificateError from .util.ssl_ import ( resolve_cert_reqs, resolve_ssl_version, - ssl_wrap_socket, assert_fingerprint, + create_urllib3_context, + ssl_wrap_socket ) @@ -174,7 +170,13 @@ class HTTPConnection(_HTTPConnection, object): """ headers = HTTPHeaderDict(headers if headers is not None else {}) skip_accept_encoding = 'accept-encoding' in headers - self.putrequest(method, url, skip_accept_encoding=skip_accept_encoding) + skip_host = 'host' in headers + self.putrequest( + method, + url, + skip_accept_encoding=skip_accept_encoding, + skip_host=skip_host + ) for header, value in headers.items(): self.putheader(header, value) if 'transfer-encoding' not in headers: @@ -203,14 +205,18 @@ class HTTPConnection(_HTTPConnection, object): class HTTPSConnection(HTTPConnection): default_port = port_by_scheme['https'] + ssl_version = None + def __init__(self, host, port=None, key_file=None, cert_file=None, - strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, **kw): + strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + ssl_context=None, **kw): HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) self.key_file = key_file self.cert_file = cert_file + self.ssl_context = ssl_context # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) @@ -219,7 +225,19 @@ class HTTPSConnection(HTTPConnection): def connect(self): conn = self._new_conn() self._prepare_conn(conn) - self.sock = ssl.wrap_socket(conn, self.key_file, self.cert_file) + + if self.ssl_context is None: + self.ssl_context = create_urllib3_context( + ssl_version=resolve_ssl_version(None), + cert_reqs=resolve_cert_reqs(None), + ) + + self.sock = ssl_wrap_socket( + sock=conn, + keyfile=self.key_file, + certfile=self.cert_file, + ssl_context=self.ssl_context, + ) class VerifiedHTTPSConnection(HTTPSConnection): @@ -237,9 +255,18 @@ class VerifiedHTTPSConnection(HTTPSConnection): cert_reqs=None, ca_certs=None, assert_hostname=None, assert_fingerprint=None, ca_cert_dir=None): - - if (ca_certs or ca_cert_dir) and cert_reqs is None: - cert_reqs = 'CERT_REQUIRED' + """ + This method should only be called once, before the connection is used. + """ + # If cert_reqs is not provided, we can try to guess. If the user gave + # us a cert database, we assume they want to use it: otherwise, if + # they gave us an SSL Context object we should use whatever is set for + # it. + if cert_reqs is None: + if ca_certs or ca_cert_dir: + cert_reqs = 'CERT_REQUIRED' + elif self.ssl_context is not None: + cert_reqs = self.ssl_context.verify_mode self.key_file = key_file self.cert_file = cert_file @@ -253,9 +280,6 @@ class VerifiedHTTPSConnection(HTTPSConnection): # Add certificate verification conn = self._new_conn() - resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs) - resolved_ssl_version = resolve_ssl_version(self.ssl_version) - hostname = self.host if getattr(self, '_tunnel_host', None): # _tunnel_host was added in Python 2.6.3 @@ -281,17 +305,27 @@ class VerifiedHTTPSConnection(HTTPSConnection): # Wrap socket using verification with the root certs in # trusted_root_certs - self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file, - cert_reqs=resolved_cert_reqs, - ca_certs=self.ca_certs, - ca_cert_dir=self.ca_cert_dir, - server_hostname=hostname, - ssl_version=resolved_ssl_version) + if self.ssl_context is None: + self.ssl_context = create_urllib3_context( + ssl_version=resolve_ssl_version(self.ssl_version), + cert_reqs=resolve_cert_reqs(self.cert_reqs), + ) + + context = self.ssl_context + context.verify_mode = resolve_cert_reqs(self.cert_reqs) + self.sock = ssl_wrap_socket( + sock=conn, + keyfile=self.key_file, + certfile=self.cert_file, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + server_hostname=hostname, + ssl_context=context) if self.assert_fingerprint: assert_fingerprint(self.sock.getpeercert(binary_form=True), self.assert_fingerprint) - elif resolved_cert_reqs != ssl.CERT_NONE \ + elif context.verify_mode != ssl.CERT_NONE \ and self.assert_hostname is not False: cert = self.sock.getpeercert() if not cert.get('subjectAltName', ()): @@ -304,8 +338,10 @@ class VerifiedHTTPSConnection(HTTPSConnection): ) _match_hostname(cert, self.assert_hostname or hostname) - self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or - self.assert_fingerprint is not None) + self.is_verified = ( + context.verify_mode == ssl.CERT_REQUIRED or + self.assert_fingerprint is not None + ) def _match_hostname(cert, asserted_hostname): diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index ab634cb4..19d08f23 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -7,13 +7,6 @@ import warnings from socket import error as SocketError, timeout as SocketTimeout import socket -try: # Python 3 - from queue import LifoQueue, Empty, Full -except ImportError: - from Queue import LifoQueue, Empty, Full - # Queue is imported for side effects on MS Windows - import Queue as _unused_module_Queue # noqa: unused - from .exceptions import ( ClosedPoolError, @@ -32,6 +25,7 @@ from .exceptions import ( ) from .packages.ssl_match_hostname import CertificateError from .packages import six +from .packages.six.moves.queue import LifoQueue, Empty, Full from .connection import ( port_by_scheme, DummyConnection, @@ -48,6 +42,10 @@ from .util.timeout import Timeout from .util.url import get_host, Url +if six.PY2: + # Queue is imported for side effects on MS Windows + import Queue as _unused_module_Queue # noqa: F401 + xrange = six.moves.xrange log = logging.getLogger(__name__) @@ -210,8 +208,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.info("Starting new HTTP connection (%d): %s", - self.num_connections, self.host) + log.debug("Starting new HTTP connection (%d): %s", + self.num_connections, self.host) conn = self.ConnectionCls(host=self.host, port=self.port, timeout=self.timeout.connect_timeout, @@ -246,7 +244,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): - log.info("Resetting dropped connection: %s", self.host) + log.debug("Resetting dropped connection: %s", self.host) conn.close() if getattr(conn, 'auto_open', 1) == 0: # This is a proxied connection that has been mutated by @@ -397,8 +395,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # AppEngine doesn't have a version attr. http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') - log.debug("\"%s %s %s\" %s %s", method, url, http_version, - httplib_response.status, httplib_response.length) + log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port, + method, url, http_version, httplib_response.status, + httplib_response.length) try: assert_header_parsing(httplib_response.msg) @@ -600,10 +599,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # mess. response_conn = conn if not release_conn else None + # Pass method to Response for length checking + response_kw['request_method'] = method + # Import httplib's response into our own wrapper object response = self.ResponseCls.from_httplib(httplib_response, pool=self, connection=response_conn, + retries=retries, **response_kw) # Everything went great! @@ -683,7 +686,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): raise return response - log.info("Redirecting %s -> %s", url, redirect_location) + retries.sleep_for_retry(response) + log.debug("Redirecting %s -> %s", url, redirect_location) return self.urlopen( method, redirect_location, body, headers, retries=retries, redirect=redirect, @@ -692,7 +696,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): release_conn=release_conn, **response_kw) # Check if we should retry the HTTP response. - if retries.is_forced_retry(method, status_code=response.status): + has_retry_after = bool(response.getheader('Retry-After')) + if retries.is_retry(method, response.status, has_retry_after): try: retries = retries.increment(method, url, response=response, _pool=self) except MaxRetryError: @@ -702,8 +707,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): response.release_conn() raise return response - retries.sleep() - log.info("Forced retry: %s", url) + retries.sleep(response) + log.debug("Retry: %s", url) return self.urlopen( method, url, body, headers, retries=retries, redirect=redirect, @@ -775,7 +780,6 @@ class HTTPSConnectionPool(HTTPConnectionPool): assert_hostname=self.assert_hostname, assert_fingerprint=self.assert_fingerprint) conn.ssl_version = self.ssl_version - return conn def _prepare_proxy(self, conn): @@ -801,8 +805,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.info("Starting new HTTPS connection (%d): %s", - self.num_connections, self.host) + log.debug("Starting new HTTPS connection (%d): %s", + self.num_connections, self.host) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: raise SSLError("Can't connect to HTTPS URL because the SSL " @@ -834,7 +838,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): warnings.warn(( 'Unverified HTTPS request is being made. ' 'Adding certificate verification is strongly advised. See: ' - 'https://urllib3.readthedocs.io/en/latest/security.html'), + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings'), InsecureRequestWarning) diff --git a/requests/packages/urllib3/contrib/appengine.py b/requests/packages/urllib3/contrib/appengine.py index 1579476c..be9262f4 100644 --- a/requests/packages/urllib3/contrib/appengine.py +++ b/requests/packages/urllib3/contrib/appengine.py @@ -1,7 +1,48 @@ +""" +This module provides a pool manager that uses Google App Engine's +`URLFetch Service `_. + +Example usage:: + + from urllib3 import PoolManager + from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox + + if is_appengine_sandbox(): + # AppEngineManager uses AppEngine's URLFetch API behind the scenes + http = AppEngineManager() + else: + # PoolManager uses a socket-level API behind the scenes + http = PoolManager() + + r = http.request('GET', 'https://google.com/') + +There are `limitations `_ to the URLFetch service and it may not be +the best choice for your application. There are three options for using +urllib3 on Google App Engine: + +1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is + cost-effective in many circumstances as long as your usage is within the + limitations. +2. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets. + Sockets also have `limitations and restrictions + `_ and have a lower free quota than URLFetch. + To use sockets, be sure to specify the following in your ``app.yaml``:: + + env_variables: + GAE_USE_SOCKETS_HTTPLIB : 'true' + +3. If you are using `App Engine Flexible +`_, you can use the standard +:class:`PoolManager` without any configuration or special environment variables. +""" + from __future__ import absolute_import import logging import os import warnings +from urlparse import urljoin from ..exceptions import ( HTTPError, @@ -41,13 +82,12 @@ class AppEngineManager(RequestMethods): This manager uses the URLFetch service directly instead of using the emulated httplib, and is subject to URLFetch limitations as described in - the App Engine documentation here: + the App Engine documentation `here + `_. - https://cloud.google.com/appengine/docs/python/urlfetch - - Notably it will raise an AppEnginePlatformError if: + Notably it will raise an :class:`AppEnginePlatformError` if: * URLFetch is not available. - * If you attempt to use this on GAEv2 (Managed VMs), as full socket + * If you attempt to use this on App Engine Flexible, as full socket support is available. * If a request size is more than 10 megabytes. * If a response size is more than 32 megabtyes. @@ -56,7 +96,8 @@ class AppEngineManager(RequestMethods): Beyond those cases, it will raise normal urllib3 errors. """ - def __init__(self, headers=None, retries=None, validate_certificate=True): + def __init__(self, headers=None, retries=None, validate_certificate=True, + urlfetch_retries=True): if not urlfetch: raise AppEnginePlatformError( "URLFetch is not available in this environment.") @@ -75,6 +116,7 @@ class AppEngineManager(RequestMethods): RequestMethods.__init__(self, headers) self.validate_certificate = validate_certificate + self.urlfetch_retries = urlfetch_retries self.retries = retries or Retry.DEFAULT @@ -92,16 +134,17 @@ class AppEngineManager(RequestMethods): retries = self._get_retries(retries, redirect) try: + follow_redirects = ( + redirect and + retries.redirect != 0 and + retries.total) response = urlfetch.fetch( url, payload=body, method=method, headers=headers or {}, allow_truncated=False, - follow_redirects=( - redirect and - retries.redirect != 0 and - retries.total), + follow_redirects=self.urlfetch_retries and follow_redirects, deadline=self._get_absolute_timeout(timeout), validate_certificate=self.validate_certificate, ) @@ -133,19 +176,40 @@ class AppEngineManager(RequestMethods): "URLFetch does not support method: %s" % method, e) http_response = self._urlfetch_response_to_http_response( - response, **response_kw) + response, retries=retries, **response_kw) - # Check for redirect response - if (http_response.get_redirect_location() and - retries.raise_on_redirect and redirect): - raise MaxRetryError(self, url, "too many redirects") + # Handle redirect? + redirect_location = redirect and http_response.get_redirect_location() + if redirect_location: + # Check for redirect response + if (self.urlfetch_retries and retries.raise_on_redirect): + raise MaxRetryError(self, url, "too many redirects") + else: + if http_response.status == 303: + method = 'GET' + + try: + retries = retries.increment(method, url, response=http_response, _pool=self) + except MaxRetryError: + if retries.raise_on_redirect: + raise MaxRetryError(self, url, "too many redirects") + return http_response + + retries.sleep_for_retry(http_response) + log.debug("Redirecting %s -> %s", url, redirect_location) + redirect_url = urljoin(url, redirect_location) + return self.urlopen( + method, redirect_url, body, headers, + retries=retries, redirect=redirect, + timeout=timeout, **response_kw) # Check if we should retry the HTTP response. - if retries.is_forced_retry(method, status_code=http_response.status): + has_retry_after = bool(http_response.getheader('Retry-After')) + if retries.is_retry(method, http_response.status, has_retry_after): retries = retries.increment( method, url, response=http_response, _pool=self) - log.info("Forced retry: %s", url) - retries.sleep() + log.debug("Retry: %s", url) + retries.sleep(http_response) return self.urlopen( method, url, body=body, headers=headers, @@ -183,12 +247,13 @@ class AppEngineManager(RequestMethods): def _get_absolute_timeout(self, timeout): if timeout is Timeout.DEFAULT_TIMEOUT: - return 5 # 5s is the default timeout for URLFetch. + return None # Defer to URLFetch's default. if isinstance(timeout, Timeout): - if timeout._read is not timeout._connect: + if timeout._read is not None or timeout._connect is not None: warnings.warn( "URLFetch does not support granular timeout settings, " - "reverting to total timeout.", AppEnginePlatformWarning) + "reverting to total or default URLFetch timeout.", + AppEnginePlatformWarning) return timeout.total return timeout diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py index 11d0b5c3..642e99ed 100644 --- a/requests/packages/urllib3/contrib/ntlmpool.py +++ b/requests/packages/urllib3/contrib/ntlmpool.py @@ -5,14 +5,11 @@ Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 """ from __future__ import absolute_import -try: - from http.client import HTTPSConnection -except ImportError: - from httplib import HTTPSConnection from logging import getLogger from ntlm import ntlm -from urllib3 import HTTPSConnectionPool +from .. import HTTPSConnectionPool +from ..packages.six.moves.http_client import HTTPSConnection log = getLogger(__name__) diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py index ed3b9cc3..c6db25d4 100644 --- a/requests/packages/urllib3/contrib/pyopenssl.py +++ b/requests/packages/urllib3/contrib/pyopenssl.py @@ -1,17 +1,21 @@ -'''SSL with SNI_-support for Python 2. Follow these instructions if you would +""" +SSL with SNI_-support for Python 2. Follow these instructions if you would like to verify SSL certificates in Python 2. Note, the default libraries do *not* do certificate checking; you need to do additional work to validate certificates yourself. This needs the following packages installed: -* pyOpenSSL (tested with 0.13) -* ndg-httpsclient (tested with 0.3.2) -* pyasn1 (tested with 0.1.6) +* pyOpenSSL (tested with 16.0.0) +* cryptography (minimum 1.3.4, from pyopenssl) +* idna (minimum 2.0, from cryptography) + +However, pyopenssl depends on cryptography, which depends on idna, so while we +use all three directly here we end up having relatively few packages required. You can install them with the following command: - pip install pyopenssl ndg-httpsclient pyasn1 + pip install pyopenssl cryptography idna To activate certificate checking, call :func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code @@ -34,45 +38,38 @@ compression in Python 2 (see `CRIME attack`_). If you want to configure the default list of supported cipher suites, you can set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable. -Module Variables ----------------- - -:var DEFAULT_SSL_CIPHER_LIST: The list of supported SSL/TLS cipher suites. - .. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication .. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) - -''' +""" from __future__ import absolute_import -try: - from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT - from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName -except SyntaxError as e: - raise ImportError(e) - +import idna import OpenSSL.SSL -from pyasn1.codec.der import decoder as der_decoder -from pyasn1.type import univ, constraint +from cryptography import x509 +from cryptography.hazmat.backends.openssl import backend as openssl_backend +from cryptography.hazmat.backends.openssl.x509 import _Certificate + from socket import timeout, error as SocketError +from io import BytesIO try: # Platform-specific: Python 2 from socket import _fileobject except ImportError: # Platform-specific: Python 3 _fileobject = None - from urllib3.packages.backports.makefile import backport_makefile + from ..packages.backports.makefile import backport_makefile +import logging import ssl import select import six +import sys -from .. import connection from .. import util __all__ = ['inject_into_urllib3', 'extract_from_urllib3'] -# SNI only *really* works if we can read the subjectAltName of certificates. -HAS_SNI = SUBJ_ALT_NAME_SUPPORT +# SNI always works. +HAS_SNI = True # Map from urllib3 to PyOpenSSL compatible parameter-values. _openssl_versions = { @@ -91,78 +88,120 @@ try: except AttributeError: pass -_openssl_verify = { +_stdlib_to_openssl_verify = { ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } - -DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS.encode('ascii') +_openssl_to_stdlib_verify = dict( + (v, k) for k, v in _stdlib_to_openssl_verify.items() +) # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 orig_util_HAS_SNI = util.HAS_SNI -orig_connection_ssl_wrap_socket = connection.ssl_wrap_socket +orig_util_SSLContext = util.ssl_.SSLContext + + +log = logging.getLogger(__name__) def inject_into_urllib3(): 'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.' - connection.ssl_wrap_socket = ssl_wrap_socket + util.ssl_.SSLContext = PyOpenSSLContext util.HAS_SNI = HAS_SNI + util.ssl_.HAS_SNI = HAS_SNI util.IS_PYOPENSSL = True + util.ssl_.IS_PYOPENSSL = True def extract_from_urllib3(): 'Undo monkey-patching by :func:`inject_into_urllib3`.' - connection.ssl_wrap_socket = orig_connection_ssl_wrap_socket + util.ssl_.SSLContext = orig_util_SSLContext util.HAS_SNI = orig_util_HAS_SNI + util.ssl_.HAS_SNI = orig_util_HAS_SNI util.IS_PYOPENSSL = False + util.ssl_.IS_PYOPENSSL = False -# Note: This is a slightly bug-fixed version of same from ndg-httpsclient. -class SubjectAltName(BaseSubjectAltName): - '''ASN.1 implementation for subjectAltNames support''' +def _dnsname_to_stdlib(name): + """ + Converts a dNSName SubjectAlternativeName field to the form used by the + standard library on the given Python version. - # There is no limit to how many SAN certificates a certificate may have, - # however this needs to have some limit so we'll set an arbitrarily high - # limit. - sizeSpec = univ.SequenceOf.sizeSpec + \ - constraint.ValueSizeConstraint(1, 1024) + Cryptography produces a dNSName as a unicode string that was idna-decoded + from ASCII bytes. We need to idna-encode that string to get it back, and + then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib + uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). + """ + def idna_encode(name): + """ + Borrowed wholesale from the Python Cryptography Project. It turns out + that we can't just safely call `idna.encode`: it can explode for + wildcard names. This avoids that problem. + """ + for prefix in [u'*.', u'.']: + if name.startswith(prefix): + name = name[len(prefix):] + return prefix.encode('ascii') + idna.encode(name) + return idna.encode(name) + + name = idna_encode(name) + if sys.version_info >= (3, 0): + name = name.decode('utf-8') + return name -# Note: This is a slightly bug-fixed version of same from ndg-httpsclient. def get_subj_alt_name(peer_cert): - # Search through extensions - dns_name = [] - if not SUBJ_ALT_NAME_SUPPORT: - return dns_name + """ + Given an PyOpenSSL certificate, provides all the subject alternative names. + """ + # Pass the cert to cryptography, which has much better APIs for this. + # This is technically using private APIs, but should work across all + # relevant versions until PyOpenSSL gets something proper for this. + cert = _Certificate(openssl_backend, peer_cert._x509) - general_names = SubjectAltName() - for i in range(peer_cert.get_extension_count()): - ext = peer_cert.get_extension(i) - ext_name = ext.get_short_name() - if ext_name != b'subjectAltName': - continue + # We want to find the SAN extension. Ask Cryptography to locate it (it's + # faster than looping in Python) + try: + ext = cert.extensions.get_extension_for_class( + x509.SubjectAlternativeName + ).value + except x509.ExtensionNotFound: + # No such extension, return the empty list. + return [] + except (x509.DuplicateExtension, x509.UnsupportedExtension, + x509.UnsupportedGeneralNameType, UnicodeError) as e: + # A problem has been found with the quality of the certificate. Assume + # no SAN field is present. + log.warning( + "A problem was encountered with the certificate that prevented " + "urllib3 from finding the SubjectAlternativeName field. This can " + "affect certificate validation. The error was %s", + e, + ) + return [] - # PyOpenSSL returns extension data in ASN.1 encoded form - ext_dat = ext.get_data() - decoded_dat = der_decoder.decode(ext_dat, - asn1Spec=general_names) + # We want to return dNSName and iPAddress fields. We need to cast the IPs + # back to strings because the match_hostname function wants them as + # strings. + # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8 + # decoded. This is pretty frustrating, but that's what the standard library + # does with certificates, and so we need to attempt to do the same. + names = [ + ('DNS', _dnsname_to_stdlib(name)) + for name in ext.get_values_for_type(x509.DNSName) + ] + names.extend( + ('IP Address', str(name)) + for name in ext.get_values_for_type(x509.IPAddress) + ) - for name in decoded_dat: - if not isinstance(name, SubjectAltName): - continue - for entry in range(len(name)): - component = name.getComponentByPosition(entry) - if component.getName() != 'dNSName': - continue - dns_name.append(str(component.getComponent())) - - return dns_name + return names class WrappedSocket(object): @@ -282,10 +321,7 @@ class WrappedSocket(object): 'subject': ( (('commonName', x509.get_subject().CN),), ), - 'subjectAltName': [ - ('DNS', value) - for value in get_subj_alt_name(x509) - ] + 'subjectAltName': get_subj_alt_name(x509) } def _reuse(self): @@ -308,51 +344,88 @@ else: # Platform-specific: Python 3 WrappedSocket.makefile = makefile +class PyOpenSSLContext(object): + """ + I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible + for translating the interface of the standard library ``SSLContext`` object + to calls into PyOpenSSL. + """ + def __init__(self, protocol): + self.protocol = _openssl_versions[protocol] + self._ctx = OpenSSL.SSL.Context(self.protocol) + self._options = 0 + self.check_hostname = False + + @property + def options(self): + return self._options + + @options.setter + def options(self, value): + self._options = value + self._ctx.set_options(value) + + @property + def verify_mode(self): + return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()] + + @verify_mode.setter + def verify_mode(self, value): + self._ctx.set_verify( + _stdlib_to_openssl_verify[value], + _verify_callback + ) + + def set_default_verify_paths(self): + self._ctx.set_default_verify_paths() + + def set_ciphers(self, ciphers): + if isinstance(ciphers, six.text_type): + ciphers = ciphers.encode('utf-8') + self._ctx.set_cipher_list(ciphers) + + def load_verify_locations(self, cafile=None, capath=None, cadata=None): + if cafile is not None: + cafile = cafile.encode('utf-8') + if capath is not None: + capath = capath.encode('utf-8') + self._ctx.load_verify_locations(cafile, capath) + if cadata is not None: + self._ctx.load_verify_locations(BytesIO(cadata)) + + def load_cert_chain(self, certfile, keyfile=None, password=None): + self._ctx.use_certificate_file(certfile) + if password is not None: + self._ctx.set_passwd_cb(lambda max_length, prompt_twice, userdata: password) + self._ctx.use_privatekey_file(keyfile or certfile) + + def wrap_socket(self, sock, server_side=False, + do_handshake_on_connect=True, suppress_ragged_eofs=True, + server_hostname=None): + cnx = OpenSSL.SSL.Connection(self._ctx, sock) + + if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 + server_hostname = server_hostname.encode('utf-8') + + if server_hostname is not None: + cnx.set_tlsext_host_name(server_hostname) + + cnx.set_connect_state() + + while True: + try: + cnx.do_handshake() + except OpenSSL.SSL.WantReadError: + rd, _, _ = select.select([sock], [], [], sock.gettimeout()) + if not rd: + raise timeout('select timed out') + continue + except OpenSSL.SSL.Error as e: + raise ssl.SSLError('bad handshake: %r' % e) + break + + return WrappedSocket(cnx, sock) + + def _verify_callback(cnx, x509, err_no, err_depth, return_code): return err_no == 0 - - -def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, - ca_certs=None, server_hostname=None, - ssl_version=None, ca_cert_dir=None): - ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version]) - if certfile: - keyfile = keyfile or certfile # Match behaviour of the normal python ssl library - ctx.use_certificate_file(certfile) - if keyfile: - ctx.use_privatekey_file(keyfile) - if cert_reqs != ssl.CERT_NONE: - ctx.set_verify(_openssl_verify[cert_reqs], _verify_callback) - if ca_certs or ca_cert_dir: - try: - ctx.load_verify_locations(ca_certs, ca_cert_dir) - except OpenSSL.SSL.Error as e: - raise ssl.SSLError('bad ca_certs: %r' % ca_certs, e) - else: - ctx.set_default_verify_paths() - - # Disable TLS compression to mitigate CRIME attack (issue #309) - OP_NO_COMPRESSION = 0x20000 - ctx.set_options(OP_NO_COMPRESSION) - - # Set list of supported ciphersuites. - ctx.set_cipher_list(DEFAULT_SSL_CIPHER_LIST) - - cnx = OpenSSL.SSL.Connection(ctx, sock) - if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 - server_hostname = server_hostname.encode('utf-8') - cnx.set_tlsext_host_name(server_hostname) - cnx.set_connect_state() - while True: - try: - cnx.do_handshake() - except OpenSSL.SSL.WantReadError: - rd, _, _ = select.select([sock], [], [], sock.gettimeout()) - if not rd: - raise timeout('select timed out') - continue - except OpenSSL.SSL.Error as e: - raise ssl.SSLError('bad handshake: %r' % e) - break - - return WrappedSocket(cnx, sock) diff --git a/requests/packages/urllib3/contrib/socks.py b/requests/packages/urllib3/contrib/socks.py index 81970fa6..c8fa8409 100644 --- a/requests/packages/urllib3/contrib/socks.py +++ b/requests/packages/urllib3/contrib/socks.py @@ -1,17 +1,23 @@ # -*- coding: utf-8 -*- """ -SOCKS support for urllib3 -~~~~~~~~~~~~~~~~~~~~~~~~~ - -This contrib module contains provisional support for SOCKS proxies from within +This module contains provisional support for SOCKS proxies from within urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and SOCKS5. To enable its functionality, either install PySocks or install this module with the ``socks`` extra. +The SOCKS implementation supports the full range of urllib3 features. It also +supports the following SOCKS features: + +- SOCKS4 +- SOCKS4a +- SOCKS5 +- Usernames and passwords for the SOCKS proxy + Known Limitations: - Currently PySocks does not support contacting remote websites via literal - IPv6 addresses. Any such connection attempt will fail. + IPv6 addresses. Any such connection attempt will fail. You must use a domain + name. - Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any such connection attempt will fail. """ diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index f2e65917..8a091c17 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -1,4 +1,7 @@ from __future__ import absolute_import +from .packages.six.moves.http_client import ( + IncompleteRead as httplib_IncompleteRead +) # Base Exceptions @@ -193,6 +196,35 @@ class ResponseNotChunked(ProtocolError, ValueError): pass +class BodyNotHttplibCompatible(HTTPError): + """ + Body should be httplib.HTTPResponse like (have an fp attribute which + returns raw chunks) for read_chunked(). + """ + pass + + +class IncompleteRead(HTTPError, httplib_IncompleteRead): + """ + Response length doesn't match expected Content-Length + + Subclass of http_client.IncompleteRead to allow int value + for `partial` to avoid creating large objects on streamed + reads. + """ + def __init__(self, partial, expected): + super(IncompleteRead, self).__init__(partial, expected) + + def __repr__(self): + return ('IncompleteRead(%i bytes read, ' + '%i more expected)' % (self.partial, self.expected)) + + +class InvalidHeader(HTTPError): + "The header provided was somehow invalid." + pass + + class ProxySchemeUnknown(AssertionError, ValueError): "ProxyManager does not support the supplied scheme" # TODO(t-8ch): Stop inheriting from AssertionError in v2.0. diff --git a/requests/packages/urllib3/fields.py b/requests/packages/urllib3/fields.py index 8fa2a127..19b0ae0c 100644 --- a/requests/packages/urllib3/fields.py +++ b/requests/packages/urllib3/fields.py @@ -130,7 +130,7 @@ class RequestField(object): iterable = header_parts.items() for name, value in iterable: - if value: + if value is not None: parts.append(self._render_part(name, value)) return '; '.join(parts) diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py index 97a2843c..cd11cee4 100644 --- a/requests/packages/urllib3/filepost.py +++ b/requests/packages/urllib3/filepost.py @@ -13,7 +13,7 @@ writer = codecs.lookup('utf-8')[3] def choose_boundary(): """ - Our embarassingly-simple replacement for mimetools.choose_boundary. + Our embarrassingly-simple replacement for mimetools.choose_boundary. """ return uuid4().hex diff --git a/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore b/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore deleted file mode 100644 index 0a764a4d..00000000 --- a/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore +++ /dev/null @@ -1 +0,0 @@ -env diff --git a/requests/packages/urllib3/packages/ssl_match_hostname/__init__.py b/requests/packages/urllib3/packages/ssl_match_hostname/__init__.py index dd59a75f..d6594eb2 100644 --- a/requests/packages/urllib3/packages/ssl_match_hostname/__init__.py +++ b/requests/packages/urllib3/packages/ssl_match_hostname/__init__.py @@ -1,5 +1,11 @@ +import sys + try: - # Python 3.2+ + # Our match_hostname function is the same as 3.5's, so we only want to + # import the match_hostname function if it's at least that good. + if sys.version_info < (3, 5): + raise ImportError("Fallback to vendored code") + from ssl import CertificateError, match_hostname except ImportError: try: diff --git a/requests/packages/urllib3/packages/ssl_match_hostname/_implementation.py b/requests/packages/urllib3/packages/ssl_match_hostname/_implementation.py index 52f42873..1fd42f38 100644 --- a/requests/packages/urllib3/packages/ssl_match_hostname/_implementation.py +++ b/requests/packages/urllib3/packages/ssl_match_hostname/_implementation.py @@ -4,8 +4,20 @@ # stdlib. http://docs.python.org/3/license.html import re +import sys + +# ipaddress has been backported to 2.6+ in pypi. If it is installed on the +# system, use it to handle IPAddress ServerAltnames (this was added in +# python-3.5) otherwise only do DNS matching. This allows +# backports.ssl_match_hostname to continue to be used all the way back to +# python-2.4. +try: + import ipaddress +except ImportError: + ipaddress = None + +__version__ = '3.5.0.1' -__version__ = '3.4.0.2' class CertificateError(ValueError): pass @@ -64,6 +76,23 @@ def _dnsname_match(dn, hostname, max_wildcards=1): return pat.match(hostname) +def _to_unicode(obj): + if isinstance(obj, str) and sys.version_info < (3,): + obj = unicode(obj, encoding='ascii', errors='strict') + return obj + +def _ipaddress_match(ipname, host_ip): + """Exact matching of IP addresses. + + RFC 6125 explicitly doesn't define an algorithm for this + (section 1.7.2 - "Out of Scope"). + """ + # OpenSSL may add a trailing newline to a subjectAltName's IP address + # Divergence from upstream: ipaddress can't handle byte str + ip = ipaddress.ip_address(_to_unicode(ipname).rstrip()) + return ip == host_ip + + def match_hostname(cert, hostname): """Verify that *cert* (in decoded format as returned by SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 @@ -73,12 +102,35 @@ def match_hostname(cert, hostname): returns nothing. """ if not cert: - raise ValueError("empty or no certificate") + raise ValueError("empty or no certificate, match_hostname needs a " + "SSL socket or SSL context with either " + "CERT_OPTIONAL or CERT_REQUIRED") + try: + # Divergence from upstream: ipaddress can't handle byte str + host_ip = ipaddress.ip_address(_to_unicode(hostname)) + except ValueError: + # Not an IP address (common case) + host_ip = None + except UnicodeError: + # Divergence from upstream: Have to deal with ipaddress not taking + # byte strings. addresses should be all ascii, so we consider it not + # an ipaddress in this case + host_ip = None + except AttributeError: + # Divergence from upstream: Make ipaddress library optional + if ipaddress is None: + host_ip = None + else: + raise dnsnames = [] san = cert.get('subjectAltName', ()) for key, value in san: if key == 'DNS': - if _dnsname_match(value, hostname): + if host_ip is None and _dnsname_match(value, hostname): + return + dnsnames.append(value) + elif key == 'IP Address': + if host_ip is not None and _ipaddress_match(value, host_ip): return dnsnames.append(value) if not dnsnames: diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index 7ed00b1c..276b54dd 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -3,15 +3,11 @@ import collections import functools import logging -try: # Python 3 - from urllib.parse import urljoin -except ImportError: - from urlparse import urljoin - from ._collections import RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .connectionpool import port_by_scheme from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown +from .packages.six.moves.urllib.parse import urljoin from .request import RequestMethods from .util.url import parse_url from .util.retry import Retry @@ -23,7 +19,7 @@ __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] log = logging.getLogger(__name__) SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', - 'ssl_version', 'ca_cert_dir') + 'ssl_version', 'ca_cert_dir', 'ssl_context') # The base fields to use when determining what pool to get a connection from; # these do not rely on the ``connection_pool_kw`` and can be determined by the diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py index d5aa62d8..c0fddff0 100644 --- a/requests/packages/urllib3/request.py +++ b/requests/packages/urllib3/request.py @@ -1,10 +1,7 @@ from __future__ import absolute_import -try: - from urllib.parse import urlencode -except ImportError: - from urllib import urlencode from .filepost import encode_multipart_formdata +from .packages.six.moves.urllib.parse import urlencode __all__ = ['RequestMethods'] diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 55679032..6f1b63c8 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -2,18 +2,22 @@ from __future__ import absolute_import from contextlib import contextmanager import zlib import io +import logging from socket import timeout as SocketTimeout from socket import error as SocketError from ._collections import HTTPHeaderDict from .exceptions import ( - ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked + BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, + ResponseNotChunked, IncompleteRead, InvalidHeader ) from .packages.six import string_types as basestring, binary_type, PY3 from .packages.six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head +log = logging.getLogger(__name__) + class DeflateDecoder(object): @@ -89,6 +93,14 @@ class HTTPResponse(io.IOBase): When this HTTPResponse wrapper is generated from an httplib.HTTPResponse object, it's convenient to include the original for debug purposes. It's otherwise unused. + + :param retries: + The retries contains the last :class:`~urllib3.util.retry.Retry` that + was used during the request. + + :param enforce_content_length: + Enforce content length checking. Body returned by server must match + value of Content-Length header, if present. Otherwise, raise error. """ CONTENT_DECODERS = ['gzip', 'deflate'] @@ -96,7 +108,8 @@ class HTTPResponse(io.IOBase): def __init__(self, body='', headers=None, status=0, version=0, reason=None, strict=0, preload_content=True, decode_content=True, - original_response=None, pool=None, connection=None): + original_response=None, pool=None, connection=None, + retries=None, enforce_content_length=False, request_method=None): if isinstance(headers, HTTPHeaderDict): self.headers = headers @@ -107,6 +120,8 @@ class HTTPResponse(io.IOBase): self.reason = reason self.strict = strict self.decode_content = decode_content + self.retries = retries + self.enforce_content_length = enforce_content_length self._decoder = None self._body = None @@ -132,6 +147,9 @@ class HTTPResponse(io.IOBase): if "chunked" in encodings: self.chunked = True + # Determine length of response + self.length_remaining = self._init_length(request_method) + # If requested, preload the body. if preload_content and not self._body: self._body = self.read(decode_content=decode_content) @@ -177,9 +195,57 @@ class HTTPResponse(io.IOBase): """ return self._fp_bytes_read + def _init_length(self, request_method): + """ + Set initial length value for Response content if available. + """ + length = self.headers.get('content-length') + + if length is not None and self.chunked: + # This Response will fail with an IncompleteRead if it can't be + # received as chunked. This method falls back to attempt reading + # the response before raising an exception. + log.warning("Received response with both Content-Length and " + "Transfer-Encoding set. This is expressly forbidden " + "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " + "attempting to process response as Transfer-Encoding: " + "chunked.") + return None + + elif length is not None: + try: + # RFC 7230 section 3.3.2 specifies multiple content lengths can + # be sent in a single Content-Length header + # (e.g. Content-Length: 42, 42). This line ensures the values + # are all valid ints and that as long as the `set` length is 1, + # all values are the same. Otherwise, the header is invalid. + lengths = set([int(val) for val in length.split(',')]) + if len(lengths) > 1: + raise InvalidHeader("Content-Length contained multiple " + "unmatching values (%s)" % length) + length = lengths.pop() + except ValueError: + length = None + else: + if length < 0: + length = None + + # Convert status to int for comparison + # In some cases, httplib returns a status of "_UNKNOWN" + try: + status = int(self.status) + except ValueError: + status = 0 + + # Check for responses that shouldn't include a body + if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD': + length = 0 + + return length + def _init_decoder(self): """ - Set-up the _decoder attribute if necessar. + Set-up the _decoder attribute if necessary. """ # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 @@ -322,9 +388,18 @@ class HTTPResponse(io.IOBase): # no harm in redundantly calling close. self._fp.close() flush_decoder = True + if self.enforce_content_length and self.length_remaining not in (0, None): + # This is an edge case that httplib failed to cover due + # to concerns of backward compatibility. We're + # addressing it here to make sure IncompleteRead is + # raised during streaming, so all calls with incorrect + # Content-Length are caught. + raise IncompleteRead(self._fp_bytes_read, self.length_remaining) if data: self._fp_bytes_read += len(data) + if self.length_remaining is not None: + self.length_remaining -= len(data) data = self._decode(data, decode_content, flush_decoder) @@ -349,7 +424,7 @@ class HTTPResponse(io.IOBase): If True, will attempt to decode the body based on the 'content-encoding' header. """ - if self.chunked: + if self.chunked and self.supports_chunked_reads(): for line in self.read_chunked(amt, decode_content=decode_content): yield line else: @@ -407,10 +482,10 @@ class HTTPResponse(io.IOBase): def closed(self): if self._fp is None: return True + elif hasattr(self._fp, 'isclosed'): + return self._fp.isclosed() elif hasattr(self._fp, 'closed'): return self._fp.closed - elif hasattr(self._fp, 'isclosed'): # Python 2 - return self._fp.isclosed() else: return True @@ -440,6 +515,15 @@ class HTTPResponse(io.IOBase): b[:len(temp)] = temp return len(temp) + def supports_chunked_reads(self): + """ + Checks if the underlying file-like object looks like a + httplib.HTTPResponse object. We do this by testing for the fp + attribute. If it is present we assume it returns raw chunks as + processed by read_chunked(). + """ + return hasattr(self._fp, 'fp') + def _update_chunk_length(self): # First, we'll figure out length of a chunk and then # we'll try to read it from socket. @@ -491,6 +575,10 @@ class HTTPResponse(io.IOBase): raise ResponseNotChunked( "Response is not chunked. " "Header 'transfer-encoding: chunked' is missing.") + if not self.supports_chunked_reads(): + raise BodyNotHttplibCompatible( + "Body should be httplib.HTTPResponse like. " + "It should have have an fp attribute which returns raw chunks.") # Don't bother reading the body of a HEAD request. if self._original_response and is_response_to_head(self._original_response): diff --git a/requests/packages/urllib3/util/response.py b/requests/packages/urllib3/util/response.py index 0b5c75c1..67cf730a 100644 --- a/requests/packages/urllib3/util/response.py +++ b/requests/packages/urllib3/util/response.py @@ -12,6 +12,13 @@ def is_fp_closed(obj): The file-like object to check. """ + try: + # Check `isclosed()` first, in case Python3 doesn't set `closed`. + # GH Issue #928 + return obj.isclosed() + except AttributeError: + pass + try: # Check via the official file-like-object way. return obj.closed diff --git a/requests/packages/urllib3/util/retry.py b/requests/packages/urllib3/util/retry.py index d379833c..47ad539a 100644 --- a/requests/packages/urllib3/util/retry.py +++ b/requests/packages/urllib3/util/retry.py @@ -1,6 +1,10 @@ from __future__ import absolute_import import time import logging +from collections import namedtuple +from itertools import takewhile +import email +import re from ..exceptions import ( ConnectTimeoutError, @@ -8,12 +12,17 @@ from ..exceptions import ( ProtocolError, ReadTimeoutError, ResponseError, + InvalidHeader, ) from ..packages import six log = logging.getLogger(__name__) +# Data structure for representing the metadata of requests that result in a retry. +RequestHistory = namedtuple('RequestHistory', ["method", "url", "error", + "status", "redirect_location"]) + class Retry(object): """ Retry configuration. @@ -113,18 +122,29 @@ class Retry(object): whether we should raise an exception, or return a response, if status falls in ``status_forcelist`` range and retries have been exhausted. + + :param tuple history: The history of the request encountered during + each call to :meth:`~Retry.increment`. The list is in the order + the requests occurred. Each list item is of class :class:`RequestHistory`. + + :param bool respect_retry_after_header: + Whether to respect Retry-After header on status codes defined as + :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. + """ DEFAULT_METHOD_WHITELIST = frozenset([ 'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']) + RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) + #: Maximum backoff time. BACKOFF_MAX = 120 def __init__(self, total=10, connect=None, read=None, redirect=None, method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, backoff_factor=0, raise_on_redirect=True, raise_on_status=True, - _observed_errors=0): + history=None, respect_retry_after_header=True): self.total = total self.connect = connect @@ -140,7 +160,8 @@ class Retry(object): self.backoff_factor = backoff_factor self.raise_on_redirect = raise_on_redirect self.raise_on_status = raise_on_status - self._observed_errors = _observed_errors # TODO: use .history instead? + self.history = history or tuple() + self.respect_retry_after_header = respect_retry_after_header def new(self, **kw): params = dict( @@ -151,7 +172,7 @@ class Retry(object): backoff_factor=self.backoff_factor, raise_on_redirect=self.raise_on_redirect, raise_on_status=self.raise_on_status, - _observed_errors=self._observed_errors, + history=self.history, ) params.update(kw) return type(self)(**params) @@ -175,23 +196,71 @@ class Retry(object): :rtype: float """ - if self._observed_errors <= 1: + # We want to consider only the last consecutive errors sequence (Ignore redirects). + consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None, + reversed(self.history)))) + if consecutive_errors_len <= 1: return 0 - backoff_value = self.backoff_factor * (2 ** (self._observed_errors - 1)) + backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1)) return min(self.BACKOFF_MAX, backoff_value) - def sleep(self): - """ Sleep between retry attempts using an exponential backoff. + def parse_retry_after(self, retry_after): + # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4 + if re.match(r"^\s*[0-9]+\s*$", retry_after): + seconds = int(retry_after) + else: + retry_date_tuple = email.utils.parsedate(retry_after) + if retry_date_tuple is None: + raise InvalidHeader("Invalid Retry-After header: %s" % retry_after) + retry_date = time.mktime(retry_date_tuple) + seconds = retry_date - time.time() - By default, the backoff factor is 0 and this method will return - immediately. - """ + if seconds < 0: + seconds = 0 + + return seconds + + def get_retry_after(self, response): + """ Get the value of Retry-After in seconds. """ + + retry_after = response.getheader("Retry-After") + + if retry_after is None: + return None + + return self.parse_retry_after(retry_after) + + def sleep_for_retry(self, response=None): + retry_after = self.get_retry_after(response) + if retry_after: + time.sleep(retry_after) + return True + + return False + + def _sleep_backoff(self): backoff = self.get_backoff_time() if backoff <= 0: return time.sleep(backoff) + def sleep(self, response=None): + """ Sleep between retry attempts. + + This method will respect a server's ``Retry-After`` response header + and sleep the duration of the time requested. If that is not present, it + will use an exponential backoff. By default, the backoff factor is 0 and + this method will return immediately. + """ + + if response: + slept = self.sleep_for_retry(response) + if slept: + return + + self._sleep_backoff() + def _is_connection_error(self, err): """ Errors when we're fairly sure that the server did not receive the request, so it should be safe to retry. @@ -204,13 +273,17 @@ class Retry(object): """ return isinstance(err, (ReadTimeoutError, ProtocolError)) - def is_forced_retry(self, method, status_code): + def is_retry(self, method, status_code, has_retry_after=False): """ Is this method/status code retryable? (Based on method/codes whitelists) """ if self.method_whitelist and method.upper() not in self.method_whitelist: return False - return self.status_forcelist and status_code in self.status_forcelist + if self.status_forcelist and status_code in self.status_forcelist: + return True + + return (self.total and self.respect_retry_after_header and + has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES)) def is_exhausted(self): """ Are we out of retries? """ @@ -241,11 +314,12 @@ class Retry(object): if total is not None: total -= 1 - _observed_errors = self._observed_errors connect = self.connect read = self.read redirect = self.redirect cause = 'unknown' + status = None + redirect_location = None if error and self._is_connection_error(error): # Connect retry? @@ -253,7 +327,6 @@ class Retry(object): raise six.reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 - _observed_errors += 1 elif error and self._is_read_error(error): # Read retry? @@ -261,27 +334,30 @@ class Retry(object): raise six.reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 - _observed_errors += 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = 'too many redirects' + redirect_location = response.get_redirect_location() + status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and a the given method is in the whitelist - _observed_errors += 1 cause = ResponseError.GENERIC_ERROR if response and response.status: cause = ResponseError.SPECIFIC_ERROR.format( status_code=response.status) + status = response.status + + history = self.history + (RequestHistory(method, url, error, status, redirect_location),) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, - _observed_errors=_observed_errors) + history=history) if new_retry.is_exhausted(): raise MaxRetryError(_pool, url, error or ResponseError(cause)) diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py index 4a64d7ef..c4c55df6 100644 --- a/requests/packages/urllib3/util/ssl_.py +++ b/requests/packages/urllib3/util/ssl_.py @@ -11,7 +11,6 @@ from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning SSLContext = None HAS_SNI = False -create_default_context = None IS_PYOPENSSL = False # Maps the length of a digest to a possible hash function producing this digest @@ -63,14 +62,25 @@ except ImportError: # The general intent is: # - Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), # - prefer ECDHE over DHE for better performance, -# - prefer any AES-GCM over any AES-CBC for better performance and security, -# - use 3DES as fallback which is secure but slow, +# - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and +# security, +# - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common, # - disable NULL authentication, MD5 MACs and DSS for security reasons. -DEFAULT_CIPHERS = ( - 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:' - 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:' - '!eNULL:!MD5' -) +DEFAULT_CIPHERS = ':'.join([ + 'ECDH+AESGCM', + 'ECDH+CHACHA20', + 'DH+AESGCM', + 'DH+CHACHA20', + 'ECDH+AES256', + 'DH+AES256', + 'ECDH+AES128', + 'DH+AES', + 'RSA+AESGCM', + 'RSA+AES', + '!aNULL', + '!eNULL', + '!MD5', +]) try: from ssl import SSLContext # Modern SSL? @@ -117,8 +127,8 @@ except ImportError: 'urllib3 from configuring SSL appropriately and may cause ' 'certain SSL connections to fail. You can upgrade to a newer ' 'version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/security.html' - '#insecureplatformwarning.', + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings', InsecurePlatformWarning ) kwargs = { @@ -287,6 +297,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, """ context = ssl_context if context is None: + # Note: This branch of code and all the variables in it are no longer + # used by urllib3 itself. We should consider deprecating and removing + # this code. context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers) @@ -301,6 +314,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if e.errno == errno.ENOENT: raise SSLError(e) raise + elif getattr(context, 'load_default_certs', None) is not None: + # try to load OS default certs; works well on Windows (require Python3.4+) + context.load_default_certs() if certfile: context.load_cert_chain(certfile, keyfile) @@ -313,8 +329,8 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, 'This may cause the server to present an incorrect TLS ' 'certificate, which can cause validation failures. You can upgrade to ' 'a newer version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/security.html' - '#snimissingwarning.', + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings', SNIMissingWarning ) return context.wrap_socket(sock) diff --git a/requests/packages/urllib3/util/timeout.py b/requests/packages/urllib3/util/timeout.py index ff62f476..1eaa6a35 100644 --- a/requests/packages/urllib3/util/timeout.py +++ b/requests/packages/urllib3/util/timeout.py @@ -111,8 +111,8 @@ class Timeout(object): :param name: The name of the timeout attribute to validate. This is used to specify in error messages. :return: The validated and casted version of the given value. - :raises ValueError: If the type is not an integer or a float, or if it - is a numeric value less than zero. + :raises ValueError: If it is a numeric value less than or equal to + zero, or the type is not an integer, float, or None. """ if value is _Default: return cls.DEFAULT_TIMEOUT @@ -120,20 +120,23 @@ class Timeout(object): if value is None or value is cls.DEFAULT_TIMEOUT: return value + if isinstance(value, bool): + raise ValueError("Timeout cannot be a boolean value. It must " + "be an int, float or None.") try: float(value) except (TypeError, ValueError): raise ValueError("Timeout value %s was %s, but it must be an " - "int or float." % (name, value)) + "int, float or None." % (name, value)) try: - if value < 0: + if value <= 0: raise ValueError("Attempted to set %s timeout to %s, but the " "timeout cannot be set to a value less " - "than 0." % (name, value)) + "than or equal to 0." % (name, value)) except TypeError: # Python 3 raise ValueError("Timeout value %s was %s, but it must be an " - "int or float." % (name, value)) + "int, float or None." % (name, value)) return value diff --git a/requests/packages/urllib3/util/url.py b/requests/packages/urllib3/util/url.py index e996204a..61a326e4 100644 --- a/requests/packages/urllib3/util/url.py +++ b/requests/packages/urllib3/util/url.py @@ -10,14 +10,19 @@ url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] class Url(namedtuple('Url', url_attrs)): """ Datastructure for representing an HTTP URL. Used as a return value for - :func:`parse_url`. + :func:`parse_url`. Both the scheme and host are normalized as they are + both case-insensitive according to RFC 3986. """ - slots = () + __slots__ = () def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None): if path and not path.startswith('/'): path = '/' + path + if scheme: + scheme = scheme.lower() + if host: + host = host.lower() return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment) @@ -184,10 +189,14 @@ def parse_url(url): host = _host if port: - # If given, ports must be integers. + # If given, ports must be integers. No whitespace, no plus or + # minus prefixes, no non-integer digits such as ^2 (superscript). if not port.isdigit(): raise LocationParseError(url) - port = int(port) + try: + port = int(port) + except ValueError: + raise LocationParseError(url) else: # Blank ports are cool, too. (rfc3986#section-3.2.3) port = None @@ -211,7 +220,7 @@ def parse_url(url): def get_host(url): """ - Deprecated. Use :func:`.parse_url` instead. + Deprecated. Use :func:`parse_url` instead. """ p = parse_url(url) return p.scheme or 'http', p.hostname, p.port