From 374ac589bd689748640303c41cd774d28700e177 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 5 Nov 2018 11:22:08 -0500 Subject: [PATCH 1/3] Fix PIP_SHIMs import, pythonfinder, vistir - Add new exceptions - Improve validation - Cleanup spinner text - Fix pyenv finder when `PYENV_ROOT` is not set - Fix import errors for users installing and using `PIP_SHIMS` in pipenv Signed-off-by: Dan Ryan --- pipenv/cli/options.py | 26 +++++++------- pipenv/core.py | 40 ++++++++++++---------- pipenv/exceptions.py | 25 +++++++++++++- pipenv/vendor/pythonfinder/environment.py | 4 ++- pipenv/vendor/pythonfinder/models/pyenv.py | 2 +- pipenv/vendor/vistir/__init__.py | 2 +- pipenv/vendor/vistir/spin.py | 4 +++ 7 files changed, 68 insertions(+), 35 deletions(-) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index dcdc1d9f..b701ef90 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import os -from click import BOOL as click_booltype +import click.types from click import ( BadParameter, Group, Option, argument, echo, make_pass_decorator, option ) @@ -103,7 +103,7 @@ def extra_index_option(f): def editable_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) - state.installstate.editables.extend(value) + state.installstate.editables.extend(validate_requirements(ctx, param, value)) return value return option('-e', '--editable', expose_value=False, multiple=True, help='An editable python package URL or path, often to a VCS repo.', @@ -117,7 +117,7 @@ def sequential_option(f): return value return option("--sequential", is_flag=True, default=False, expose_value=False, help="Install dependencies one-at-a-time, instead of concurrently.", - callback=callback, type=click_booltype)(f) + callback=callback, type=click.types.BOOL)(f) def skip_lock_option(f): @@ -127,7 +127,7 @@ def skip_lock_option(f): return value return option("--skip-lock", is_flag=True, default=False, expose_value=False, help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", - callback=callback, type=click_booltype)(f) + callback=callback, type=click.types.BOOL)(f) def keep_outdated_option(f): @@ -137,7 +137,7 @@ def keep_outdated_option(f): return value return option("--keep-outdated", is_flag=True, default=False, expose_value=False, help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", - callback=callback, type=click_booltype)(f) + callback=callback, type=click.types.BOOL)(f) def selective_upgrade_option(f): @@ -145,7 +145,7 @@ def selective_upgrade_option(f): state = ctx.ensure_object(State) state.installstate.selective_upgrade = value return value - return option("--selective-upgrade", is_flag=True, default=False, type=click_booltype, + return option("--selective-upgrade", is_flag=True, default=False, type=click.types.BOOL, help="Update specified packages.", callback=callback, expose_value=False)(f) @@ -165,7 +165,7 @@ def dev_option(f): state = ctx.ensure_object(State) state.installstate.dev = value return value - return option("--dev", "-d", is_flag=True, default=False, type=click_booltype, + return option("--dev", "-d", is_flag=True, default=False, type=click.types.BOOL, help="Install both develop and default packages.", callback=callback, expose_value=False)(f) @@ -176,13 +176,13 @@ def pre_option(f): state.installstate.pre = value return value return option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.", - callback=callback, type=click_booltype, expose_value=False)(f) + callback=callback, type=click.types.BOOL, expose_value=False)(f) def package_arg(f): def callback(ctx, param, value): state = ctx.ensure_object(State) - state.installstate.packages.extend(value) + state.installstate.packages.extend(validate_requirements(ctx, param, value)) return value return argument('packages', nargs=-1, callback=callback, expose_value=False,)(f) @@ -235,7 +235,7 @@ def site_packages_option(f): state = ctx.ensure_object(State) state.site_packages = value return value - return option("--site-packages", is_flag=True, default=False, type=click_booltype, + return option("--site-packages", is_flag=True, default=False, type=click.types.BOOL, help="Enable site-packages for the virtualenv.", callback=callback, expose_value=False)(f) @@ -245,7 +245,7 @@ def clear_option(f): state = ctx.ensure_object(State) state.clear = value return value - return option("--clear", is_flag=True, callback=callback, type=click_booltype, + return option("--clear", is_flag=True, callback=callback, type=click.types.BOOL, help="Clears caches (pipenv, pip, and pip-tools).", expose_value=False)(f) @@ -257,7 +257,7 @@ def system_option(f): state.system = value return value return option("--system", is_flag=True, default=False, help="System pip management.", - callback=callback, type=click_booltype, expose_value=False)(f) + callback=callback, type=click.types.BOOL, expose_value=False)(f) def requirementstxt_option(f): @@ -295,7 +295,7 @@ def deploy_option(f): state = ctx.ensure_object(State) state.installstate.deploy = value return value - return option("--deploy", is_flag=True, default=False, type=click_booltype, + return option("--deploy", is_flag=True, default=False, type=click.types.BOOL, help=u"Abort if the Pipfile.lock is out-of-date, or Python version is" " wrong.", callback=callback, expose_value=False)(f) diff --git a/pipenv/core.py b/pipenv/core.py index bf4145e5..7aa38b41 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1105,17 +1105,18 @@ def do_lock( # Support for --keep-outdated… if keep_outdated: + from pipenv.vendor.packaging.utils import canonicalize_name for section_name, section in ( ("default", project.packages), ("develop", project.dev_packages), ): - for package_specified in section: - norm_name = pep423_name(package_specified) + for package_specified in section.keys(): if not is_pinned(section[package_specified]): - if norm_name in cached_lockfile[section_name]: - lockfile[section_name][norm_name] = cached_lockfile[ + canonical_name = canonicalize_name(package_specified) + if canonical_name in cached_lockfile[section_name]: + lockfile[section_name][canonical_name] = cached_lockfile[ section_name - ][norm_name] + ][canonical_name].copy() # Overwrite any develop packages with default packages. for default_package in lockfile["default"]: if default_package in lockfile["develop"]: @@ -1756,9 +1757,11 @@ def do_install( if requirements or package_args or project.pipfile_exists: skip_requirements = True # Don't attempt to install develop and default packages if Pipfile is missing - if not project.pipfile_exists and not packages and dev: - click.echo("Could not find Pipfile.", err=True) - sys.exit(1) + if not project.pipfile_exists and not (packages or dev) and not code: + if not (skip_lock or deploy): + raise exceptions.PipfileNotFound(project.pipfile_location) + elif (skip_lock or deploy) and not project.lockfile_exists: + raise exceptions.LockfileNotFound(project.lockfile_location) concurrent = not sequential # Ensure that virtualenv is available. ensure_project( @@ -1778,15 +1781,7 @@ def do_install( remote = requirements and is_valid_url(requirements) # Warn and exit if --system is used without a pipfile. if (system and package_args) and not (PIPENV_VIRTUALENV): - click.echo( - "{0}: --system is intended to be used for Pipfile installation, " - "not installation of specific packages. Aborting.".format( - crayons.red("Warning", bold=True) - ), - err=True, - ) - click.echo("See also: --deploy flag.", err=True) - sys.exit(1) + raise exceptions.SystemUsageError # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True @@ -1923,6 +1918,7 @@ def do_install( pre=pre, requirements_dir=requirements_directory, pypi_mirror=pypi_mirror, + keep_outdated=keep_outdated ) # This is for if the user passed in dependencies, then we want to maek sure we @@ -1952,6 +1948,7 @@ def do_install( sys.exit(1) if index_url: pkg_requirement.index = index_url + try: c = pip_install( pkg_requirement, ignore_hashes=True, @@ -1964,6 +1961,9 @@ def do_install( extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) + except (ValueError, RuntimeError): + sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) # Warn if --editable wasn't passed. if pkg_requirement.is_vcs and not pkg_requirement.editable: sp.write_err( @@ -2215,7 +2215,8 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror click.echo(fix_utf8("Launching subshell in virtual environment…"), err=True) fork_args = (project.virtualenv_location, project.project_directory, shell_args) - + with vistir.contextmanagers.temp_environ(): + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) if fancy: shell.fork(*fork_args) return @@ -2349,6 +2350,9 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") load_dot_env() # Activate virtualenv under the current interpreter's environment + + with vistir.contextmanagers.temp_environ(): + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) inline_activate_virtual_environment() try: script = project.build_script(command, args) diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 95feb859..3fda8021 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -176,6 +176,19 @@ class PipenvOptionsError(PipenvUsageError): self.option_name = option_name +class SystemUsageError(PipenvOptionsError): + def __init__(self, option_name="system", message=None, ctx=None, **kwargs): + extra = kwargs.pop("extra", []) + extra += [ + "{0}: --system is intended to be used for Pipfile installation, " + "not installation of specific packages. Aborting.".format( + crayons.red("Warning", bold=True) + ), + ] + message = crayons.blue("See also: {0}".format(crayons.white("-deploy flag."))) + super(SystemUsageError, self).__init__(option_name, message=message, ctx=ctx, extra=extra, **kwargs) + + class PipfileException(PipenvFileError): def __init__(self, hint=None, **kwargs): from .core import project @@ -239,6 +252,16 @@ class UninstallError(PipenvException): self.extra = extra +class InstallError(PipenvException): + def __init__(self, package, **kwargs): + message = "{0} {1}".format( + crayons.red("ERROR:", bold=True), + crayons.yellow("Package installation failed...") + ) + extra = kwargs.pop("extra", []) + PipenvException.__init__(self, message=fix_utf8(message), extra=extra, **kwargs) + + class CacheError(PipenvException): def __init__(self, path, **kwargs): message = "{0} {1} {2}\n{0}".format( @@ -273,7 +296,7 @@ class ResolutionFailure(PipenvException): crayons.red("ERROR:", bold=True), crayons.yellow(message) ) if no_version_found: - messsage = "{0}\n{1}".format( + message = "{0}\n{1}".format( message, crayons.blue( "Please check your version specifier and version number. " diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index 7c69b9fc..27a5b3fc 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -7,7 +7,9 @@ import sys PYENV_INSTALLED = bool(os.environ.get("PYENV_SHELL")) or bool( os.environ.get("PYENV_ROOT") ) -PYENV_ROOT = os.path.expandvars(os.environ.get("PYENV_ROOT", "~/.pyenv")) +PYENV_ROOT = os.path.expanduser( + os.path.expandvars(os.environ.get("PYENV_ROOT", "~/.pyenv")) +) IS_64BIT_OS = None SYSTEM_ARCH = platform.architecture()[0] diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 5874af64..4a8dfc65 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -71,7 +71,7 @@ class PyenvFinder(BaseFinder, BasePath): @versions.default def get_versions(self): versions = defaultdict() - bin_ = sysconfig._INSTALL_SCHEMES[sysconfig._get_default_scheme()]["scripts"] + bin_ = sysconfig._INSTALL_SCHEMES['posix_prefix']["scripts"] for p in self.get_version_order(): bin_dir = Path(bin_.format(base=p.as_posix())) version_path = None diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index c8a995fa..f3554d56 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -31,7 +31,7 @@ from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfi from .spin import VistirSpinner, create_spinner -__version__ = "0.2.3" +__version__ = "0.2.4" __all__ = [ diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index 20587d9d..f8c4e009 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -153,11 +153,15 @@ class VistirSpinner(base_obj): def ok(self, text="OK"): """Set Ok (success) finalizer to a spinner.""" + self._text = None + _text = text if text else "OK" self._freeze(_text) def fail(self, text="FAIL"): """Set fail finalizer to a spinner.""" + self._text = None + _text = text if text else "FAIL" self._freeze(_text) From 6b2f3b66a65b0966da48ce612995fb0ee6a36da9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 5 Nov 2018 11:30:41 -0500 Subject: [PATCH 2/3] Syntax error fix Signed-off-by: Dan Ryan --- pipenv/core.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index d03ba042..26ac4d0e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1949,18 +1949,18 @@ def do_install( if index_url: pkg_requirement.index = index_url try: - c = pip_install( - pkg_requirement, - ignore_hashes=True, - allow_global=system, - selective_upgrade=selective_upgrade, - no_deps=False, - pre=pre, - requirements_dir=requirements_directory, - index=index_url, - extra_indexes=extra_index_url, - pypi_mirror=pypi_mirror, - ) + c = pip_install( + pkg_requirement, + ignore_hashes=True, + allow_global=system, + selective_upgrade=selective_upgrade, + no_deps=False, + pre=pre, + requirements_dir=requirements_directory, + index=index_url, + extra_indexes=extra_index_url, + pypi_mirror=pypi_mirror, + ) except (ValueError, RuntimeError): sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) From 363ffc64085bd13749970052201738b5e807eebc Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 5 Nov 2018 11:36:33 -0500 Subject: [PATCH 3/3] Remove unfinished validation Signed-off-by: Dan Ryan --- pipenv/cli/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index b701ef90..277c014a 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -103,7 +103,7 @@ def extra_index_option(f): def editable_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) - state.installstate.editables.extend(validate_requirements(ctx, param, value)) + state.installstate.editables.extend(value) return value return option('-e', '--editable', expose_value=False, multiple=True, help='An editable python package URL or path, often to a VCS repo.', @@ -182,7 +182,7 @@ def pre_option(f): def package_arg(f): def callback(ctx, param, value): state = ctx.ensure_object(State) - state.installstate.packages.extend(validate_requirements(ctx, param, value)) + state.installstate.packages.extend(value) return value return argument('packages', nargs=-1, callback=callback, expose_value=False,)(f)