From 4c8617237ccdf53bcf87607479afbd9e8cee99bb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 21 Oct 2018 23:54:30 -0400 Subject: [PATCH] Update requirementslib, requests and vistir Signed-off-by: Dan Ryan --- pipenv/vendor/requests/__init__.py | 17 +-- pipenv/vendor/requests/__version__.py | 4 +- pipenv/vendor/requests/adapters.py | 27 +++-- pipenv/vendor/requests/api.py | 20 ++-- pipenv/vendor/requests/auth.py | 4 +- pipenv/vendor/requests/compat.py | 3 +- pipenv/vendor/requests/cookies.py | 31 ++--- pipenv/vendor/requests/help.py | 3 +- pipenv/vendor/requests/hooks.py | 4 +- pipenv/vendor/requests/models.py | 17 +-- pipenv/vendor/requests/sessions.py | 52 ++++++--- pipenv/vendor/requests/status_codes.py | 2 +- pipenv/vendor/requests/utils.py | 19 ++-- .../requirementslib/models/dependencies.py | 11 +- .../vendor/requirementslib/models/lockfile.py | 49 ++++++-- .../vendor/requirementslib/models/pipfile.py | 78 +++++++++++-- .../requirementslib/models/requirements.py | 4 +- pipenv/vendor/requirementslib/utils.py | 106 ++++++++++++++++++ pipenv/vendor/vistir/contextmanagers.py | 2 +- pipenv/vendor/vistir/misc.py | 55 ++++++++- 20 files changed, 393 insertions(+), 115 deletions(-) diff --git a/pipenv/vendor/requests/__init__.py b/pipenv/vendor/requests/__init__.py index a5b3c9c3..bc168ee5 100644 --- a/pipenv/vendor/requests/__init__.py +++ b/pipenv/vendor/requests/__init__.py @@ -22,7 +22,7 @@ usage: ... or POST: >>> payload = dict(key1='value1', key2='value2') - >>> r = requests.post('http://httpbin.org/post', data=payload) + >>> r = requests.post('https://httpbin.org/post', data=payload) >>> print(r.text) { ... @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.23 + # urllib3 >= 1.21.1, <= 1.24 assert major == 1 assert minor >= 21 - assert minor <= 23 + assert minor <= 24 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] @@ -79,14 +79,14 @@ def _check_cryptography(cryptography_version): return if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warning = 'Old version of cryptography ({}) may cause slowdown.'.format(cryptography_version) warnings.warn(warning, RequestsDependencyWarning) # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) except (AssertionError, ValueError): - warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported " + warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " "version!".format(urllib3.__version__, chardet.__version__), RequestsDependencyWarning) @@ -123,12 +123,7 @@ from .exceptions import ( # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/pipenv/vendor/requests/__version__.py b/pipenv/vendor/requests/__version__.py index ef61ec0f..be8a45fe 100644 --- a/pipenv/vendor/requests/__version__.py +++ b/pipenv/vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.19.1' -__build__ = 0x021901 +__version__ = '2.20.0' +__build__ = 0x022000 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/pipenv/vendor/requests/adapters.py b/pipenv/vendor/requests/adapters.py index a4b02842..fa4d9b3c 100644 --- a/pipenv/vendor/requests/adapters.py +++ b/pipenv/vendor/requests/adapters.py @@ -26,6 +26,7 @@ from urllib3.exceptions import ProtocolError from urllib3.exceptions import ReadTimeoutError from urllib3.exceptions import SSLError as _SSLError from urllib3.exceptions import ResponseError +from urllib3.exceptions import LocationValueError from .models import Response from .compat import urlparse, basestring @@ -35,7 +36,8 @@ from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL, + InvalidURL) from .auth import _basic_auth_str try: @@ -127,8 +129,7 @@ class HTTPAdapter(BaseAdapter): self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in - self.__attrs__) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because @@ -224,7 +225,7 @@ class HTTPAdapter(BaseAdapter): if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) + "invalid path: {}".format(cert_loc)) conn.cert_reqs = 'CERT_REQUIRED' @@ -246,10 +247,10 @@ class HTTPAdapter(BaseAdapter): conn.key_file = None if conn.cert_file and not os.path.exists(conn.cert_file): raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(conn.cert_file)) + "invalid path: {}".format(conn.cert_file)) if conn.key_file and not os.path.exists(conn.key_file): raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(conn.key_file)) + "invalid path: {}".format(conn.key_file)) def build_response(self, req, resp): """Builds a :class:`Response ` object from a urllib3 @@ -378,7 +379,7 @@ class HTTPAdapter(BaseAdapter): when subclassing the :class:`HTTPAdapter `. - :param proxies: The url of the proxy being used for this request. + :param proxy: The url of the proxy being used for this request. :rtype: dict """ headers = {} @@ -407,7 +408,10 @@ class HTTPAdapter(BaseAdapter): :rtype: requests.Response """ - conn = self.get_connection(request.url, proxies) + try: + conn = self.get_connection(request.url, proxies) + except LocationValueError as e: + raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) @@ -421,7 +425,7 @@ class HTTPAdapter(BaseAdapter): timeout = TimeoutSauce(connect=connect, read=read) except ValueError as e: # this may raise a string formatting error. - err = ("Invalid timeout {0}. Pass a (connect, read) " + err = ("Invalid timeout {}. Pass a (connect, read) " "timeout tuple, or a single float to set " "both timeouts to the same value".format(timeout)) raise ValueError(err) @@ -471,11 +475,10 @@ class HTTPAdapter(BaseAdapter): # Receive the response from the server try: - # For Python 2.7+ versions, use buffering of HTTP - # responses + # For Python 2.7, use buffering of HTTP responses r = low_conn.getresponse(buffering=True) except TypeError: - # For compatibility with Python 2.6 versions and back + # For compatibility with Python 3.3+ r = low_conn.getresponse() resp = HTTPResponse.from_httplib( diff --git a/pipenv/vendor/requests/api.py b/pipenv/vendor/requests/api.py index a2cc84d7..abada96d 100644 --- a/pipenv/vendor/requests/api.py +++ b/pipenv/vendor/requests/api.py @@ -18,8 +18,10 @@ def request(method, url, **kwargs): :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. @@ -47,7 +49,7 @@ def request(method, url, **kwargs): Usage:: >>> import requests - >>> req = requests.request('GET', 'http://httpbin.org/get') + >>> req = requests.request('GET', 'https://httpbin.org/get') """ @@ -62,7 +64,8 @@ def get(url, params=None, **kwargs): r"""Sends a GET request. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response @@ -102,7 +105,8 @@ def post(url, data=None, json=None, **kwargs): r"""Sends a POST request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -116,7 +120,8 @@ def put(url, data=None, **kwargs): r"""Sends a PUT request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -130,7 +135,8 @@ def patch(url, data=None, **kwargs): r"""Sends a PATCH request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object diff --git a/pipenv/vendor/requests/auth.py b/pipenv/vendor/requests/auth.py index 4ae45947..bdde51c7 100644 --- a/pipenv/vendor/requests/auth.py +++ b/pipenv/vendor/requests/auth.py @@ -38,7 +38,7 @@ def _basic_auth_str(username, password): if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(username), category=DeprecationWarning, @@ -48,7 +48,7 @@ def _basic_auth_str(username, password): if not isinstance(password, basestring): warnings.warn( "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(password), category=DeprecationWarning, diff --git a/pipenv/vendor/requests/compat.py b/pipenv/vendor/requests/compat.py index 6b9c6fac..c44b35ef 100644 --- a/pipenv/vendor/requests/compat.py +++ b/pipenv/vendor/requests/compat.py @@ -43,9 +43,8 @@ if is_py2: import cookielib from Cookie import Morsel from StringIO import StringIO - from collections import Callable, Mapping, MutableMapping + from collections import Callable, Mapping, MutableMapping, OrderedDict - from urllib3.packages.ordered_dict import OrderedDict builtin_str = str bytes = str diff --git a/pipenv/vendor/requests/cookies.py b/pipenv/vendor/requests/cookies.py index 50883a84..56fccd9c 100644 --- a/pipenv/vendor/requests/cookies.py +++ b/pipenv/vendor/requests/cookies.py @@ -444,20 +444,21 @@ def create_cookie(name, value, **kwargs): By default, the pair of `name` and `value` will be set for the domain '' and sent on every request (this is sometimes called a "supercookie"). """ - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False,) + result = { + 'version': 0, + 'name': name, + 'value': value, + 'port': None, + 'domain': '', + 'path': '/', + 'secure': False, + 'expires': None, + 'discard': True, + 'comment': None, + 'comment_url': None, + 'rest': {'HttpOnly': None}, + 'rfc2109': False, + } badargs = set(kwargs) - set(result) if badargs: @@ -511,6 +512,7 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): :param cookiejar: (optional) A cookiejar to add the cookies to. :param overwrite: (optional) If False, will not replace cookies already in the jar with new ones. + :rtype: CookieJar """ if cookiejar is None: cookiejar = RequestsCookieJar() @@ -529,6 +531,7 @@ def merge_cookies(cookiejar, cookies): :param cookiejar: CookieJar object to add the cookies to. :param cookies: Dictionary or CookieJar object to be added. + :rtype: CookieJar """ if not isinstance(cookiejar, cookielib.CookieJar): raise ValueError('You can only merge into CookieJar') diff --git a/pipenv/vendor/requests/help.py b/pipenv/vendor/requests/help.py index 06e06b2a..e53d35ef 100644 --- a/pipenv/vendor/requests/help.py +++ b/pipenv/vendor/requests/help.py @@ -89,8 +89,7 @@ def info(): 'version': getattr(idna, '__version__', ''), } - # OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module. - system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None) + system_ssl = ssl.OPENSSL_VERSION_NUMBER system_ssl_info = { 'version': '%x' % system_ssl if system_ssl is not None else '' } diff --git a/pipenv/vendor/requests/hooks.py b/pipenv/vendor/requests/hooks.py index 32b32de7..7a51f212 100644 --- a/pipenv/vendor/requests/hooks.py +++ b/pipenv/vendor/requests/hooks.py @@ -15,14 +15,14 @@ HOOKS = ['response'] def default_hooks(): - return dict((event, []) for event in HOOKS) + return {event: [] for event in HOOKS} # TODO: response is the only one def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" - hooks = hooks or dict() + hooks = hooks or {} hooks = hooks.get(key) if hooks: if hasattr(hooks, '__call__'): diff --git a/pipenv/vendor/requests/models.py b/pipenv/vendor/requests/models.py index 3d0e1f42..3dded57e 100644 --- a/pipenv/vendor/requests/models.py +++ b/pipenv/vendor/requests/models.py @@ -204,9 +204,13 @@ class Request(RequestHooksMixin): :param url: URL to send. :param headers: dictionary of headers to send. :param files: dictionary of {filename: fileobject} files to multipart upload. - :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. + :param data: the body to attach to the request. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param json: json for the body to attach to the request (if files or data is not specified). - :param params: dictionary of URL parameters to append to the URL. + :param params: URL parameters to append to the URL. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param auth: Auth handler or (user, pass) tuple. :param cookies: dictionary or CookieJar of cookies to attach to this request. :param hooks: dictionary of callback hooks, for internal usage. @@ -214,7 +218,7 @@ class Request(RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> req.prepare() """ @@ -274,7 +278,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> r = req.prepare() @@ -648,10 +652,7 @@ class Response(object): if not self._content_consumed: self.content - return dict( - (attr, getattr(self, attr, None)) - for attr in self.__attrs__ - ) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): diff --git a/pipenv/vendor/requests/sessions.py b/pipenv/vendor/requests/sessions.py index ba135268..a448bd83 100644 --- a/pipenv/vendor/requests/sessions.py +++ b/pipenv/vendor/requests/sessions.py @@ -115,6 +115,22 @@ class SessionRedirectMixin(object): return to_native_string(location, 'utf8') return None + def should_strip_auth(self, old_url, new_url): + """Decide whether Authorization header should be removed when redirecting""" + old_parsed = urlparse(old_url) + new_parsed = urlparse(new_url) + if old_parsed.hostname != new_parsed.hostname: + return True + # Special case: allow http -> https redirect when using the standard + # ports. This isn't specified by RFC 7235, but is kept to avoid + # breaking backwards compatibility with older versions of requests + # that allowed any redirects on the same host. + if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) + and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): + return False + # Standard case: root URI must match + return old_parsed.port != new_parsed.port or old_parsed.scheme != new_parsed.scheme + def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): """Receives a Response. Returns a generator of Responses or Requests.""" @@ -236,14 +252,10 @@ class SessionRedirectMixin(object): headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: + if 'Authorization' in headers and self.should_strip_auth(response.request.url, url): # If we get redirected to a new host, we should strip out any # authentication headers. - original_parsed = urlparse(response.request.url) - redirect_parsed = urlparse(url) - - if (original_parsed.hostname != redirect_parsed.hostname): - del headers['Authorization'] + del headers['Authorization'] # .netrc might have more auth for us on our new host. new_auth = get_netrc_auth(url) if self.trust_env else None @@ -299,7 +311,7 @@ class SessionRedirectMixin(object): """ method = prepared_request.method - # http://tools.ietf.org/html/rfc7231#section-6.4.4 + # https://tools.ietf.org/html/rfc7231#section-6.4.4 if response.status_code == codes.see_other and method != 'HEAD': method = 'GET' @@ -325,13 +337,13 @@ class Session(SessionRedirectMixin): >>> import requests >>> s = requests.Session() - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') Or as a context manager:: >>> with requests.Session() as s: - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') """ @@ -453,8 +465,8 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary, bytes, or file-like object to send - in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the @@ -550,7 +562,8 @@ class Session(SessionRedirectMixin): r"""Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response @@ -562,7 +575,8 @@ class Session(SessionRedirectMixin): r"""Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -573,7 +587,8 @@ class Session(SessionRedirectMixin): r"""Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -723,7 +738,7 @@ class Session(SessionRedirectMixin): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state def __setstate__(self, state): @@ -735,7 +750,12 @@ def session(): """ Returns a :class:`Session` for context-management. + .. deprecated:: 1.0.0 + + This method has been deprecated since version 1.0.0 and is only kept for + backwards compatibility. New code should use :class:`~requests.sessions.Session` + to create a session. This may be removed at a future date. + :rtype: Session """ - return Session() diff --git a/pipenv/vendor/requests/status_codes.py b/pipenv/vendor/requests/status_codes.py index ff462c6c..813e8c4e 100644 --- a/pipenv/vendor/requests/status_codes.py +++ b/pipenv/vendor/requests/status_codes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" +r""" The ``codes`` object defines a mapping from common names for HTTP statuses to their numerical codes, accessible either as attributes or as dictionary items. diff --git a/pipenv/vendor/requests/utils.py b/pipenv/vendor/requests/utils.py index 431f6be0..0ce7fe11 100644 --- a/pipenv/vendor/requests/utils.py +++ b/pipenv/vendor/requests/utils.py @@ -173,10 +173,10 @@ def get_netrc_auth(url, raise_errors=False): for f in NETRC_FILES: try: - loc = os.path.expanduser('~/{0}'.format(f)) + loc = os.path.expanduser('~/{}'.format(f)) except KeyError: # os.path.expanduser can fail when $HOME is undefined and - # getpwuid fails. See http://bugs.python.org/issue20164 & + # getpwuid fails. See https://bugs.python.org/issue20164 & # https://github.com/requests/requests/issues/1846 return @@ -466,7 +466,7 @@ def _parse_content_type_header(header): if index_of_equals != -1: key = param[:index_of_equals].strip(items_to_strip) value = param[index_of_equals + 1:].strip(items_to_strip) - params_dict[key] = value + params_dict[key.lower()] = value return content_type, params_dict @@ -706,6 +706,10 @@ def should_bypass_proxies(url, no_proxy): no_proxy = get_proxy('no_proxy') parsed = urlparse(url) + if parsed.hostname is None: + # URLs don't always have hostnames, e.g. file:/// urls. + return True + if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the hostname, both with and without the port. @@ -725,7 +729,7 @@ def should_bypass_proxies(url, no_proxy): else: host_with_port = parsed.hostname if parsed.port: - host_with_port += ':{0}'.format(parsed.port) + host_with_port += ':{}'.format(parsed.port) for host in no_proxy: if parsed.hostname.endswith(host) or host_with_port.endswith(host): @@ -733,13 +737,8 @@ def should_bypass_proxies(url, no_proxy): # to apply the proxies on this URL. return True - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - # The proxy_bypass function is incredibly buggy on OS X in early versions - # of Python 2.6, so allow this call to fail. Only catch the specific - # exceptions we've seen, though: this call failing in other ways can reveal - # legitimate problems. with set_environ('no_proxy', no_proxy_arg): + # parsed.hostname can be `None` in cases such as a file URI. try: bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index ae643517..d9f1b653 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -17,9 +17,10 @@ from pip_shims import ( FormatControl, InstallRequirement, PackageFinder, RequirementPreparer, RequirementSet, RequirementTracker, Resolver, WheelCache, pip_version ) -from vistir.compat import JSONDecodeError, TemporaryDirectory, fs_str +from vistir.compat import JSONDecodeError, fs_str from vistir.contextmanagers import cd, temp_environ from vistir.misc import partialclass +from vistir.path import create_tracked_tempdir from ..utils import get_pip_command, prepare_pip_source_args, _ensure_dir from .cache import CACHE_DIR, DependencyCache @@ -580,12 +581,12 @@ def start_resolver(finder=None, wheel_cache=None): download_dir = PKGS_DOWNLOAD_DIR _ensure_dir(download_dir) - _build_dir = TemporaryDirectory(fs_str("build")) - _source_dir = TemporaryDirectory(fs_str("source")) + _build_dir = create_tracked_tempdir(fs_str("build")) + _source_dir = create_tracked_tempdir(fs_str("source")) preparer = partialclass( RequirementPreparer, - build_dir=_build_dir.name, - src_dir=_source_dir.name, + build_dir=_build_dir, + src_dir=_source_dir, download_dir=download_dir, wheel_download_dir=WHEEL_DOWNLOAD_DIR, progress_bar="off", diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index bd76ca01..1997fc1f 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -14,6 +14,7 @@ from .project import ProjectFile from .requirements import Requirement from .utils import optional_instance_of +from ..utils import is_vcs, is_editable, merge_items DEFAULT_NEWLINES = u"\n" @@ -49,6 +50,27 @@ class Lockfile(object): def _get_lockfile(self): return self.projectfile.lockfile + def __getitem__(self, k, *args, **kwargs): + retval = None + lockfile = self._lockfile + section = None + pkg_type = None + try: + retval = lockfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(lockfile.get(section, {}), "_data", {}) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + def __getattr__(self, k, *args, **kwargs): retval = None lockfile = super(Lockfile, self).__getattribute__("_lockfile") @@ -56,9 +78,18 @@ class Lockfile(object): return super(Lockfile, self).__getattribute__(k) except AttributeError: retval = getattr(lockfile, k, None) - if not retval: - retval = super(Lockfile, self).__getattribute__(k, *args, **kwargs) - return retval + if retval is not None: + return retval + return super(Lockfile, self).__getattribute__(k, *args, **kwargs) + + def get_deps(self, dev=False, only=True): + deps = {} + if dev: + deps.update(self.develop._data) + if only: + return deps + deps = merge_items([deps, self.default._data]) + return deps @classmethod def read_projectfile(cls, path): @@ -135,7 +166,7 @@ class Lockfile(object): def default(self): return self._lockfile.default - def get_requirements(self, dev=False): + def get_requirements(self, dev=True, only=False): """Produces a generator which generates requirements from the desired section. :param bool dev: Indicates whether to use dev requirements, defaults to False @@ -143,20 +174,20 @@ class Lockfile(object): :rtype: :class:`~requirementslib.models.requirements.Requirement` """ - section = self.develop if dev else self.default - for k in section.keys(): - yield Requirement.from_pipfile(k, section[k]._data) + deps = self.get_deps(dev=dev, only=only) + for k, v in deps.items(): + yield Requirement.from_pipfile(k, v) @property def dev_requirements(self): if not self._dev_requirements: - self._dev_requirements = list(self.get_requirements(dev=True)) + self._dev_requirements = list(self.get_requirements(dev=True, only=True)) return self._dev_requirements @property def requirements(self): if not self._requirements: - self._requirements = list(self.get_requirements(dev=False)) + self._requirements = list(self.get_requirements(dev=False, only=True)) return self._requirements @property diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 94e9a2a1..0d1c04c8 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -6,12 +6,15 @@ import attr import copy import os +import tomlkit + from vistir.compat import Path, FileNotFoundError from .requirements import Requirement from .project import ProjectFile from .utils import optional_instance_of from ..exceptions import RequirementError +from ..utils import is_vcs, is_editable, merge_items import plette.pipfiles @@ -20,6 +23,31 @@ is_path = optional_instance_of(Path) is_projectfile = optional_instance_of(ProjectFile) +class PipfileLoader(plette.pipfiles.Pipfile): + @classmethod + def validate(cls, data): + for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items(): + if key not in data or key == "source": + continue + klass.validate(data[key]) + + @classmethod + def load(cls, f, encoding=None): + content = f.read() + if encoding is not None: + content = content.decode(encoding) + _data = tomlkit.loads(content) + if "source" not in _data: + # HACK: There is no good way to prepend a section to an existing + # TOML document, but there's no good way to copy non-structural + # content from one TOML document to another either. Modify the + # TOML content directly, and load the new in-memory document. + sep = "" if content.startswith("\n") else "\n" + content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content + data = tomlkit.loads(content) + return cls(data) + + @attr.s(slots=True) class Pipfile(object): path = attr.ib(validator=is_path, type=Path) @@ -40,16 +68,50 @@ class Pipfile(object): def _get_pipfile(self): return self.projectfile.model + @property + def pipfile(self): + return self._pipfile + + def get_deps(self, dev=False, only=True): + deps = {} + if dev: + deps.update(self.pipfile["dev-packages"]._data) + if only: + return deps + deps = merge_items([deps, self.pipfile["packages"]._data]) + return deps + + def __getitem__(self, k, *args, **kwargs): + retval = None + pipfile = self._pipfile + section = None + pkg_type = None + try: + retval = pipfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(pipfile.get(section, {}), "_data", {}) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + def __getattr__(self, k, *args, **kwargs): retval = None pipfile = super(Pipfile, self).__getattribute__("_pipfile") try: - return super(Pipfile, self).__getattribute__(k) + retval = super(Pipfile, self).__getattribute__(k) except AttributeError: retval = getattr(pipfile, k, None) - if not retval: - retval = super(Pipfile, self).__getattribute__(k, *args, **kwargs) - return retval + if retval is not None: + return retval + return super(Pipfile, self).__getattribute__(k, *args, **kwargs) @property def requires_python(self): @@ -69,7 +131,7 @@ class Pipfile(object): """ pf = ProjectFile.read( path, - plette.pipfiles.Pipfile, + PipfileLoader, invalid_ok=True ) return pf @@ -88,7 +150,7 @@ class Pipfile(object): if not path: raise RuntimeError("Must pass a path to classmethod 'Pipfile.load'") if not isinstance(path, Path): - path = Path(path) + path = Path(path).absolute() pipfile_path = path if path.name == "Pipfile" else path.joinpath("Pipfile") project_path = pipfile_path.parent if not project_path.exists(): @@ -113,10 +175,10 @@ class Pipfile(object): projectfile = cls.load_projectfile(path, create=create) pipfile = projectfile.model dev_requirements = [ - Requirement.from_pipfile(k, v._data) for k, v in pipfile.get("dev-packages", {}).items() + Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("dev-packages", {}).items() ] requirements = [ - Requirement.from_pipfile(k, v._data) for k, v in pipfile.get("packages", {}).items() + Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("packages", {}).items() ] creation_args = { "projectfile": projectfile, diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index c2768417..db004869 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -22,11 +22,11 @@ from pip_shims.shims import ( ) from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote -from vistir.compat import FileNotFoundError, Path, TemporaryDirectory +from vistir.compat import FileNotFoundError, Path from vistir.misc import dedup from vistir.path import ( create_tracked_tempdir, get_converted_relative_path, is_file_url, - is_valid_url, mkdir_p + is_valid_url ) from ..exceptions import RequirementError diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index b490d3cf..abc89831 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -5,9 +5,15 @@ import contextlib import logging import os +import boltons.iterutils import six import tomlkit +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) +from six.moves import Mapping, Sequence, Set, ItemsView from six.moves.urllib.parse import urlparse, urlsplit from pip_shims.shims import ( @@ -18,6 +24,8 @@ from vistir.compat import Path from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir + + VCS_LIST = ("git", "svn", "hg", "bzr") VCS_SCHEMES = [] SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") @@ -66,6 +74,12 @@ def is_vcs(pipfile_entry): return False +def is_editable(pipfile_entry): + if isinstance(pipfile_entry, Mapping): + return pipfile_entry.get("editable", False) is True + return False + + def multi_split(s, split): """Splits on multiple given separators.""" for r in split: @@ -181,3 +195,95 @@ def ensure_setup_py(base_dir): finally: if is_new: setup_py.unlink() + + +# Modified from https://github.com/mahmoud/boltons/blob/master/boltons/iterutils.py +def dict_path_enter(path, key, value): + if isinstance(value, six.string_types): + return value, False + elif isinstance(value, (Mapping, dict)): + return value.__class__(), ItemsView(value) + elif isinstance(value, tomlkit.items.Array): + return value.__class__([], value.trivia), enumerate(value) + elif isinstance(value, (Sequence, list)): + return value.__class__(), enumerate(value) + elif isinstance(value, (Set, set)): + return value.__class__(), enumerate(value) + else: + return value, False + + +def dict_path_exit(path, key, old_parent, new_parent, new_items): + ret = new_parent + if isinstance(new_parent, (Mapping, dict)): + vals = dict(new_items) + try: + new_parent.update(new_items) + except AttributeError: + # Handle toml containers specifically + try: + new_parent.update(vals) + # Now use default fallback if needed + except AttributeError: + ret = new_parent.__class__(vals) + elif isinstance(new_parent, tomlkit.items.Array): + vals = tomlkit.items.item([v for i, v in new_items]) + try: + new_parent._value.extend(vals._value) + except AttributeError: + ret = tomlkit.items.item(vals) + elif isinstance(new_parent, (Sequence, list)): + vals = [v for i, v in new_items] + try: + new_parent.extend(vals) + except AttributeError: + ret = new_parent.__class__(vals) # tuples + elif isinstance(new_parent, (Set, set)): + vals = [v for i, v in new_items] + try: + new_parent.update(vals) + except AttributeError: + ret = new_parent.__class__(vals) # frozensets + else: + raise RuntimeError('unexpected iterable type: %r' % type(new_parent)) + return ret + + +def merge_items(target_list, sourced=False): + if not sourced: + target_list = [(id(t), t) for t in target_list] + + ret = None + source_map = {} + + def remerge_enter(path, key, value): + new_parent, new_items = dict_path_enter(path, key, value) + if ret and not path and key is None: + new_parent = ret + + try: + cur_val = boltons.iterutils.get_path(ret, path + (key,)) + except KeyError as ke: + pass + else: + new_parent = cur_val + + return new_parent, new_items + + def remerge_exit(path, key, old_parent, new_parent, new_items): + return dict_path_exit(path, key, old_parent, new_parent, new_items) + + for t_name, target in target_list: + if sourced: + def remerge_visit(path, key, value): + source_map[path + (key,)] = t_name + return True + else: + remerge_visit = boltons.iterutils.default_visit + + ret = boltons.iterutils.remap(target, enter=remerge_enter, visit=remerge_visit, + exit=remerge_exit) + + if not sourced: + return ret + return ret, source_map diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 70f95c59..8f25e079 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -129,7 +129,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): ) else: spinner_name = None - if not start_text: + if not start_text and nospin is False: start_text = "Running..." with spinner_func( spinner_name=spinner_name, diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index f42b4ad1..e2f85985 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -10,6 +10,7 @@ import sys from collections import OrderedDict from functools import partial +from itertools import islice import six @@ -32,6 +33,9 @@ __all__ = [ "to_text", "to_bytes", "locale_encoding", + "chunked", + "take", + "divide" ] @@ -283,7 +287,10 @@ def run( cmd = Script.parse(cmd) if block or not return_object: combine_stderr = False - with spinner(spinner_name=spinner_name, start_text="Running...", nospin=nospin) as sp: + start_text = "Running..." + if nospin: + start_text = None + with spinner(spinner_name=spinner_name, start_text=start_text, nospin=nospin) as sp: return _create_subprocess( cmd, env=_env, @@ -427,6 +434,52 @@ def to_text(string, encoding="utf-8", errors=None): return string +def divide(n, iterable): + """ + split an iterable into n groups, per https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + :return: a list of new iterables derived from the original iterable + :rtype: list + """ + + seq = tuple(iterable) + q, r = divmod(len(seq), n) + + ret = [] + for i in range(n): + start = (i * q) + (i if i < r else r) + stop = ((i + 1) * q) + (i + 1 if i + 1 < r else r) + ret.append(iter(seq[start:stop])) + + return ret + + +def take(n, iterable): + """Take n elements from the supplied iterable without consuming it. + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + + from https://github.com/erikrose/more-itertools/blob/master/more_itertools/recipes.py + """ + + return list(islice(iterable, n)) + + +def chunked(n, iterable): + """Split an iterable into lists of length *n*. + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + + from https://github.com/erikrose/more-itertools/blob/master/more_itertools/more.py + """ + + return iter(partial(take, n, iter(iterable)), []) + + try: locale_encoding = locale.getdefaultencoding()[1] or "ascii" except Exception: