From f40d46274195cbf40994263494f73eb71d7b8300 Mon Sep 17 00:00:00 2001 From: zbentley Date: Wed, 6 May 2020 15:45:41 -0400 Subject: [PATCH 1/9] Allow pyenv/asdf to be used even if they are not on PATH --- pipenv/core.py | 53 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 3b549c35..dc5d25ca 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -351,6 +351,42 @@ def find_a_system_python(line): return python_entry +def find_python_installer(name, env_var): + """ + Given a python installer (pyenv or asdf), try to locate the binary for that + installer. + + pyenv/asdf are not always present on PATH. Both installers also support a + custom environment variable (PYENV_ROOT or ASDF_DIR) which alows them to + be installed into a non-default location (the default/suggested source + install location is in ~/.pyenv or ~/.asdf). + + For systems without the installers on PATH, and with a custom location + (e.g. /opt/pyenv), Pipenv can use those installers without modifications to + PATH, and only with their respective environment variables in a .env file. + This is desirable, since setting PATH in an .env file is annoying due to + the common need to reference the pre-existing PATH variable, which is not + supported in .env. + + This function searches for installer binaries using PATH, their respective + environment variables, and their respective default install locations. This + allows Pipenv to use those installers regardless of shell configuration, so + long as PYENV_ROOT or ASDF_DIR is specified in an .env file. + """ + for candidate in ( + # Look for the Python installer using the equivalent of 'which'. On + # Homebrew-installed systems, the env var may not be set, but this + # strategy will work. + find_windows_executable('', name), + # Check for explicitly set install locations (e.g. PYENV_ROOT, ASDF_DIR). + os.path.join(os.path.expanduser(os.getenv(env_var, '/dev/null')), 'bin', name), + # Check the pyenv/asdf-recommended from-source install locations + os.path.join(os.path.expanduser('~/.{}'.format(name)), 'bin', name), + ): + if candidate is not None and os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + + def ensure_python(three=None, python=None): # Support for the PIPENV_PYTHON environment variable. from .environments import PIPENV_PYTHON @@ -395,18 +431,20 @@ def ensure_python(three=None, python=None): err=True, ) # check for python installers - from .vendor.pythonfinder.environment import PYENV_INSTALLED, ASDF_INSTALLED from .installers import Pyenv, Asdf, InstallerError # prefer pyenv if both pyenv and asdf are installed as it's # dedicated to python installs so probably the preferred # method of the user for new python installs. - if PYENV_INSTALLED and not PIPENV_DONT_USE_PYENV: - installer = Pyenv("pyenv") - elif ASDF_INSTALLED and not PIPENV_DONT_USE_ASDF: - installer = Asdf("asdf") - else: - installer = None + installer = None + if not PIPENV_DONT_USE_PYENV: + pyenv_path = find_python_installer('pyenv', 'PYENV_ROOT') + if pyenv_path is not None: + installer = Pyenv(pyenv_path) + if installer is None and not PIPENV_DONT_USE_ASDF: + asdf_path = find_python_installer('asdf', 'ASDF_DIR') + if asdf_path is not None: + installer = Asdf(asdf_path) if not installer: abort() @@ -2932,3 +2970,4 @@ def do_clean( if c.return_code != 0: failure = True sys.exit(int(failure)) + From 1b91f149573c7e93e133a82a675703cee6d14b39 Mon Sep 17 00:00:00 2001 From: zbentley Date: Wed, 6 May 2020 16:56:25 -0400 Subject: [PATCH 2/9] Add news --- news/4245.feature.rst | 1 + pipenv/core.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 news/4245.feature.rst diff --git a/news/4245.feature.rst b/news/4245.feature.rst new file mode 100644 index 00000000..a78e2e66 --- /dev/null +++ b/news/4245.feature.rst @@ -0,0 +1 @@ +Pyenv/asdf can now be used whether or not they are available on PATH. Setting PYENV_ROOT/ASDF_DIR in a Pipenv's .env allows Pipenv to install an interpreter without any shell customizations, so long as pyenv/asdf is installed. diff --git a/pipenv/core.py b/pipenv/core.py index dc5d25ca..74104b6e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2970,4 +2970,3 @@ def do_clean( if c.return_code != 0: failure = True sys.exit(int(failure)) - From 7dc463306623dbd51b6d25afdb8bef04fab60f88 Mon Sep 17 00:00:00 2001 From: zbentley Date: Thu, 7 May 2020 17:51:49 -0400 Subject: [PATCH 3/9] Missed a checkin; add load_dot_env to install subcommand --- pipenv/cli/command.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 008ec479..ef52b8b6 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -226,7 +226,9 @@ def install( **kwargs ): """Installs provided packages and adds them to Pipfile, or (if no packages are given), installs all packages from Pipfile.""" - from ..core import do_install + from ..core import do_install, load_dot_env + + load_dot_env() retcode = do_install( dev=state.installstate.dev, From 64fe4df9ee3347a2d54dce308e16430df243b02b Mon Sep 17 00:00:00 2001 From: zbentley Date: Thu, 7 May 2020 20:03:33 -0400 Subject: [PATCH 4/9] Fix PYENV_YES variable --- pipenv/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 74104b6e..7cda2957 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -164,7 +164,7 @@ def load_dot_env(): err=True, ) dotenv.load_dotenv(dotenv_file, override=True) - + reload(environments) def add_to_path(p): """Adds a given path to the PATH.""" @@ -388,8 +388,8 @@ def find_python_installer(name, env_var): def ensure_python(three=None, python=None): - # Support for the PIPENV_PYTHON environment variable. - from .environments import PIPENV_PYTHON + # Runtime import is necessary due to the possibility that the environments module may have been reloaded. + from .environments import PIPENV_PYTHON, PIPENV_YES if PIPENV_PYTHON and python is False and three is None: python = PIPENV_PYTHON From b40ebc86440345d42bcbb6dd122890f0336b2555 Mon Sep 17 00:00:00 2001 From: zbentley Date: Thu, 7 May 2020 20:11:15 -0400 Subject: [PATCH 5/9] Use 'six' reload polyfill --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 7cda2957..71452957 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -164,7 +164,7 @@ def load_dot_env(): err=True, ) dotenv.load_dotenv(dotenv_file, override=True) - reload(environments) + six.moves.reload_module(environments) def add_to_path(p): """Adds a given path to the PATH.""" From bbdb3236d030af776195fc6892bf31cf230f56cc Mon Sep 17 00:00:00 2001 From: zbentley Date: Tue, 12 May 2020 18:01:49 -0400 Subject: [PATCH 6/9] Address reviewer comments --- pipenv/core.py | 63 +++++++++------------------------------- pipenv/installers.py | 68 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 75 insertions(+), 56 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 71452957..8f76fd19 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -351,42 +351,6 @@ def find_a_system_python(line): return python_entry -def find_python_installer(name, env_var): - """ - Given a python installer (pyenv or asdf), try to locate the binary for that - installer. - - pyenv/asdf are not always present on PATH. Both installers also support a - custom environment variable (PYENV_ROOT or ASDF_DIR) which alows them to - be installed into a non-default location (the default/suggested source - install location is in ~/.pyenv or ~/.asdf). - - For systems without the installers on PATH, and with a custom location - (e.g. /opt/pyenv), Pipenv can use those installers without modifications to - PATH, and only with their respective environment variables in a .env file. - This is desirable, since setting PATH in an .env file is annoying due to - the common need to reference the pre-existing PATH variable, which is not - supported in .env. - - This function searches for installer binaries using PATH, their respective - environment variables, and their respective default install locations. This - allows Pipenv to use those installers regardless of shell configuration, so - long as PYENV_ROOT or ASDF_DIR is specified in an .env file. - """ - for candidate in ( - # Look for the Python installer using the equivalent of 'which'. On - # Homebrew-installed systems, the env var may not be set, but this - # strategy will work. - find_windows_executable('', name), - # Check for explicitly set install locations (e.g. PYENV_ROOT, ASDF_DIR). - os.path.join(os.path.expanduser(os.getenv(env_var, '/dev/null')), 'bin', name), - # Check the pyenv/asdf-recommended from-source install locations - os.path.join(os.path.expanduser('~/.{}'.format(name)), 'bin', name), - ): - if candidate is not None and os.path.isfile(candidate) and os.access(candidate, os.X_OK): - return candidate - - def ensure_python(three=None, python=None): # Runtime import is necessary due to the possibility that the environments module may have been reloaded. from .environments import PIPENV_PYTHON, PIPENV_YES @@ -394,9 +358,10 @@ def ensure_python(three=None, python=None): if PIPENV_PYTHON and python is False and three is None: python = PIPENV_PYTHON - def abort(): + def abort(msg=''): click.echo( - "You can specify specific versions of Python with:\n {0}".format( + "{0}\nYou can specify specific versions of Python with:\n{1}".format( + crayons.red(msg), crayons.red( "$ pipenv --python {0}".format( os.sep.join(("path", "to", "python")) @@ -431,23 +396,25 @@ def ensure_python(three=None, python=None): err=True, ) # check for python installers - from .installers import Pyenv, Asdf, InstallerError + from .installers import Pyenv, Asdf, InstallerError, InstallerNotFound # prefer pyenv if both pyenv and asdf are installed as it's # dedicated to python installs so probably the preferred # method of the user for new python installs. installer = None if not PIPENV_DONT_USE_PYENV: - pyenv_path = find_python_installer('pyenv', 'PYENV_ROOT') - if pyenv_path is not None: - installer = Pyenv(pyenv_path) + try: + installer = Pyenv() + except InstallerNotFound: + pass if installer is None and not PIPENV_DONT_USE_ASDF: - asdf_path = find_python_installer('asdf', 'ASDF_DIR') - if asdf_path is not None: - installer = Asdf(asdf_path) + try: + installer = Pyenv() + except InstallerNotFound: + pass if not installer: - abort() + abort("Neither 'pyenv' nor 'asdf' could be found to install Python.") else: if SESSION_IS_INTERACTIVE or PIPENV_YES: try: @@ -455,9 +422,7 @@ def ensure_python(three=None, python=None): except ValueError: abort() except InstallerError as e: - click.echo(fix_utf8("Something went wrong…")) - click.echo(crayons.blue(e.err), err=True) - abort() + abort('Something went wrong while installing Python:\n{}'.format(e.err)) s = "{0} {1} {2}".format( "Would you like us to install", crayons.green("CPython {0}".format(version)), diff --git a/pipenv/installers.py b/pipenv/installers.py index cd8e49a2..70bf97d5 100644 --- a/pipenv/installers.py +++ b/pipenv/installers.py @@ -1,5 +1,7 @@ import operator import re +from abc import ABCMeta, abstractmethod + from .environments import PIPENV_INSTALL_TIMEOUT from .vendor import attr, delegator @@ -48,6 +50,10 @@ class Version(object): return (self.major, self.minor) == (other.major, other.minor) +class InstallerNotFound(RuntimeError): + pass + + class InstallerError(RuntimeError): def __init__(self, desc, c): super(InstallerError, self).__init__(desc) @@ -56,12 +62,52 @@ class InstallerError(RuntimeError): class Installer(object): + __metaclass__ = ABCMeta - def __init__(self, cmd): - self._cmd = cmd + def __init__(self): + self._cmd = self._find_installer() + super(Installer, self).__init__() - def __str__(self): - return self._cmd + @abstractmethod + def _find_installer(self): + pass + + @staticmethod + def _find_python_installer_by_name_and_env(name, env_var): + """ + Given a python installer (pyenv or asdf), try to locate the binary for that + installer. + + pyenv/asdf are not always present on PATH. Both installers also support a + custom environment variable (PYENV_ROOT or ASDF_DIR) which alows them to + be installed into a non-default location (the default/suggested source + install location is in ~/.pyenv or ~/.asdf). + + For systems without the installers on PATH, and with a custom location + (e.g. /opt/pyenv), Pipenv can use those installers without modifications to + PATH, if an installer's respective environment variable is present in an + environment's .env file. + + This function searches for installer binaries in the following locations, + by precedence: + 1. On PATH, equivalent to which(1). + 2. In the "bin" subdirectory of PYENV_ROOT or ASDF_DIR, depending on the + installer. + 3. In ~/.pyenv/bin or ~/.asdf/bin, depending on the installer. + """ + for candidate in ( + # Look for the Python installer using the equivalent of 'which'. On + # Homebrew-installed systems, the env var may not be set, but this + # strategy will work. + find_windows_executable('', name), + # Check for explicitly set install locations (e.g. PYENV_ROOT, ASDF_DIR). + os.path.join(os.path.expanduser(os.getenv(env_var, '/dev/null')), 'bin', name), + # Check the pyenv/asdf-recommended from-source install locations + os.path.join(os.path.expanduser('~/.{}'.format(name)), 'bin', name), + ): + if candidate is not None and os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + raise InstallerNotFound() def _run(self, *args, **kwargs): timeout = kwargs.pop('timeout', delegator.TIMEOUT) @@ -72,13 +118,14 @@ class Installer(object): c = delegator.run(args, block=False, timeout=timeout) c.block() if c.return_code != 0: - raise InstallerError('faild to run {0}'.format(args), c) + raise InstallerError('failed to run {0}'.format(args), c) return c + @abstractmethod def iter_installable_versions(self): """Iterate through CPython versions available for Pipenv to install. """ - raise NotImplementedError + pass def find_version_to_install(self, name): """Find a version in the installer from the version supplied. @@ -100,6 +147,7 @@ class Installer(object): ) return best_match + @abstractmethod def install(self, version): """Install the given version with runner implementation. @@ -109,11 +157,14 @@ class Installer(object): A ValueError is raised if the given version does not have a match in the runner. A InstallerError is raised if the runner command fails. """ - raise NotImplementedError + pass class Pyenv(Installer): + def _find_installer(self): + return self._find_python_installer_by_name_and_env('pyenv', 'PYENV_ROOT') + def iter_installable_versions(self): """Iterate through CPython versions available for Pipenv to install. """ @@ -140,6 +191,9 @@ class Pyenv(Installer): class Asdf(Installer): + def _find_installer(self): + return self._find_python_installer_by_name_and_env('asdf', 'ASDF_DIR') + def iter_installable_versions(self): """Iterate through CPython versions available for asdf to install. """ From 9334872fb8a5261c31ae2e2bb762fd29c5ce3109 Mon Sep 17 00:00:00 2001 From: zbentley Date: Tue, 12 May 2020 18:24:12 -0400 Subject: [PATCH 7/9] Missed imports, testing in an IDB is bad for you --- pipenv/installers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipenv/installers.py b/pipenv/installers.py index 70bf97d5..0b88160f 100644 --- a/pipenv/installers.py +++ b/pipenv/installers.py @@ -1,3 +1,4 @@ +import os import operator import re from abc import ABCMeta, abstractmethod @@ -5,6 +6,7 @@ from abc import ABCMeta, abstractmethod from .environments import PIPENV_INSTALL_TIMEOUT from .vendor import attr, delegator +from .utils import find_windows_executable @attr.s From 1f0e064f3afe67bb17ba5c2a4d82b437ec31dab0 Mon Sep 17 00:00:00 2001 From: zbentley Date: Thu, 14 May 2020 13:53:00 -0400 Subject: [PATCH 8/9] Fix misnamed asdf invocation, and debug print --- pipenv/core.py | 4 ++-- pipenv/installers.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 8f76fd19..db3abaf6 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -409,7 +409,7 @@ def ensure_python(three=None, python=None): pass if installer is None and not PIPENV_DONT_USE_ASDF: try: - installer = Pyenv() + installer = Asdf() except InstallerNotFound: pass @@ -437,7 +437,7 @@ def ensure_python(three=None, python=None): u"{0} {1} {2} {3}{4}".format( crayons.normal(u"Installing", bold=True), crayons.green(u"CPython {0}".format(version), bold=True), - crayons.normal(u"with {0}".format(installer), bold=True), + crayons.normal(u"with {0}".format(installer.cmd), bold=True), crayons.normal(u"(this may take a few minutes)"), crayons.normal(fix_utf8("…"), bold=True), ) diff --git a/pipenv/installers.py b/pipenv/installers.py index 0b88160f..f16cdacd 100644 --- a/pipenv/installers.py +++ b/pipenv/installers.py @@ -67,7 +67,7 @@ class Installer(object): __metaclass__ = ABCMeta def __init__(self): - self._cmd = self._find_installer() + self.cmd = self._find_installer() super(Installer, self).__init__() @abstractmethod @@ -116,7 +116,7 @@ class Installer(object): if kwargs: k = list(kwargs.keys())[0] raise TypeError('unexpected keyword argument {0!r}'.format(k)) - args = (self._cmd,) + tuple(args) + args = (self.cmd,) + tuple(args) c = delegator.run(args, block=False, timeout=timeout) c.block() if c.return_code != 0: From 0f999f553333f17e3f13dab5be34b7c6d00f1eef Mon Sep 17 00:00:00 2001 From: zbentley Date: Thu, 14 May 2020 17:20:39 -0400 Subject: [PATCH 9/9] Remove dotenv sourcing --- pipenv/cli/command.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index ef52b8b6..008ec479 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -226,9 +226,7 @@ def install( **kwargs ): """Installs provided packages and adds them to Pipfile, or (if no packages are given), installs all packages from Pipfile.""" - from ..core import do_install, load_dot_env - - load_dot_env() + from ..core import do_install retcode = do_install( dev=state.installstate.dev,