Fix bugs in environment implementation

- Fix virtualenv
- Update pythonfinder

Signed-off-by: Dan Ryan <dan@danryan.co>

pipenv/core.py

@@ -926,7 +926,8 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None):
         prefix=project.get_location_for_virtualenv(),
         is_venv=True,
         sources=sources,
-        pipfile=project.parsed_pipfile
+        pipfile=project.parsed_pipfile,
+        project=project
     )
     project._environment.add_dist("pipenv")
     # Say where the virtualenv is.
@@ -1626,7 +1627,7 @@ def do_outdated(pypi_mirror=None):
     outdated_packages = {
         canonicalize_name(pkg.project_name): package_info
         (pkg.project_name, pkg.parsed_version, pkg.latest_version)
-        for pkg in project.get_outdated_packages()
+        for pkg in project.environment.get_outdated_packages()
     }
     for result in installed_packages:
         dep = Requirement.from_line(str(result.as_requirement()))

pipenv/environment.py

@@ -22,7 +22,8 @@ BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path)
 
 
 class Environment(object):
-    def __init__(self, prefix=None, is_venv=False, base_working_set=None, pipfile=None, sources=None):
+    def __init__(self, prefix=None, is_venv=False, base_working_set=None, pipfile=None,
+                 sources=None, project=None):
         super(Environment, self).__init__()
         self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv}
         self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET
@@ -30,10 +31,17 @@ class Environment(object):
         self.is_venv = not prefix == os.path.normcase(os.path.normpath(sys.prefix))
         if not sources:
             sources = []
+        self.project = project
+        if project and not sources:
+            sources = project.sources
         self.sources = sources
+        if project and not pipfile:
+            pipfile = project.pipfile
+        self.pipfile = pipfile
         self.extra_dists = []
         prefix = prefix if prefix else sys.prefix
         self.prefix = vistir.compat.Path(prefix)
+        self.sys_paths = get_paths()
 
     def safe_import(self, name):
         """Helper utility for reimporting previously imported modules while inside the env"""
@@ -73,7 +81,7 @@ class Environment(object):
         deps.add(dist)
         try:
             reqs = dist.requires()
-        except AttributeError:
+        except (AttributeError, OSError):  # The METADATA file can't be found
             return deps
         for req in reqs:
             dist = working_set.find(req)
@@ -187,12 +195,6 @@ class Environment(object):
         path = json.loads(path.strip())
         return path
 
-    @cached_property
-    def system_paths(self):
-        paths = {}
-        paths = get_paths()
-        return paths
-
     @cached_property
     def sys_prefix(self):
         """The prefix run inside the context of the environment
@@ -271,7 +273,8 @@ class Environment(object):
         packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)]
         return packages
 
-    def get_finder(self):
+    @contextlib.contextmanager
+    def get_finder(self, pre=False):
         from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder
         from .environments import PIPENV_CACHE_DIR
         index_urls = [source.get("url") for source in self.sources]
@@ -286,10 +289,10 @@ class Environment(object):
         cmd_opts = pip_command.cmd_opts
         pip_command.parser.insert_option_group(0, index_opts)
         pip_command.parser.insert_option_group(0, cmd_opts)
-        pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources, [])
+        pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources)
         pip_options, _ = pip_command.parser.parse_args(pip_args)
         pip_options.cache_dir = PIPENV_CACHE_DIR
-        pip_options.pre = self.pipfile.get("pre", False)
+        pip_options.pre = self.pipfile.get("pre", pre)
         with pip_command._build_session(pip_options) as session:
             finder = PackageFinder(
                 find_links=pip_options.find_links,
@@ -300,7 +303,7 @@ class Environment(object):
             )
             yield finder
 
-    def get_package_info(self):
+    def get_package_info(self, pre=False):
         dependency_links = []
         packages = self.get_installed_packages()
         # This code is borrowed from pip's current implementation
@@ -314,7 +317,7 @@ class Environment(object):
             for dist in packages:
                 typ = 'unknown'
                 all_candidates = finder.find_all_candidates(dist.key)
-                if not finder.pip_options.pre:
+                if not self.pipfile.get("pre", finder.allow_all_prereleases):
                     # Remove prereleases
                     all_candidates = [
                         candidate for candidate in all_candidates
@@ -334,9 +337,9 @@ class Environment(object):
                 dist.latest_filetype = typ
                 yield dist
 
-    def get_outdated_packages(self):
+    def get_outdated_packages(self, pre=False):
         return [
-            pkg for pkg in self.get_package_info()
+            pkg for pkg in self.get_package_info(pre=pre)
             if pkg.latest_version._version > pkg.parsed_version._version
         ]
 

pipenv/project.py

@@ -352,13 +352,14 @@ class Project(object):
             is_venv = prefix == sys.prefix
             sources = self.sources if self.sources else [DEFAULT_SOURCE,]
             self._environment = Environment(
-                prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile
+                prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile,
+                project=self
             )
             self._environment.add_dist("pipenv")
         return self._environment
 
     def get_outdated_packages(self):
-        return self.environment.get_outdated_packages()
+        return self.environment.get_outdated_packages(pre=self.pipfile.get("pre", False))
 
     @classmethod
     def _sanitize(cls, name):

pipenv/vendor/pythonfinder/environment.py

@@ -7,9 +7,13 @@ import sys
 PYENV_INSTALLED = bool(os.environ.get("PYENV_SHELL")) or bool(
     os.environ.get("PYENV_ROOT")
 )
+ASDF_INSTALLED = bool(os.environ.get("ASDF_DATA_DIR"))
 PYENV_ROOT = os.path.expanduser(
     os.path.expandvars(os.environ.get("PYENV_ROOT", "~/.pyenv"))
 )
+ASDF_DATA_DIR = os.path.expanduser(
+    os.path.expandvars(os.environ.get("ASDF_DATA_DIR", "~/.asdf"))
+)
 IS_64BIT_OS = None
 SYSTEM_ARCH = platform.architecture()[0]
 

pipenv/vendor/pythonfinder/models/asdf.py

@@ -0,0 +1,9 @@
+# -*- coding=utf-8 -*-
+import attr
+
+from .pyenv import PyenvFinder
+
+
+@attr.s
+class AsdfFinder(PyenvFinder):
+    version_root = attr.ib(default="installs/python/*")

pipenv/vendor/pythonfinder/models/path.py

@@ -17,7 +17,7 @@ from cached_property import cached_property
 from vistir.compat import Path, fs_str
 
 from .mixins import BasePath
-from ..environment import PYENV_INSTALLED, PYENV_ROOT
+from ..environment import PYENV_INSTALLED, PYENV_ROOT, ASDF_INSTALLED, ASDF_DATA_DIR
 from ..exceptions import InvalidPythonVersion
 from ..utils import (
     ensure_path,
@@ -40,6 +40,7 @@ class SystemPath(object):
     python_version_dict = attr.ib(default=attr.Factory(defaultdict))
     only_python = attr.ib(default=False)
     pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath"))
+    asdf_finder = attr.ib(default=None)
     system = attr.ib(default=False)
     _version_dict = attr.ib(default=attr.Factory(defaultdict))
     ignore_unsupported = attr.ib(default=False)
@@ -105,6 +106,8 @@ class SystemPath(object):
             self._setup_windows()
         if PYENV_INSTALLED:
             self._setup_pyenv()
+        if ASDF_INSTALLED:
+            self._setup_asdf()
         venv = os.environ.get("VIRTUAL_ENV")
         if os.name == "nt":
             bin_dir = "Scripts"
@@ -124,32 +127,62 @@ class SystemPath(object):
                 path=syspath_bin, is_root=True, only_python=False
             )
 
-    def _setup_pyenv(self):
-        from .pyenv import PyenvFinder
-
-        last_pyenv = next(
-            (p for p in reversed(self.path_order) if PYENV_ROOT.lower() in p.lower()),
+    def _get_last_instance(self, path):
+        last_instance = next(iter(
+            (p for p in reversed(self.path_order) if path.lower() in p.lower())),
             None,
         )
         try:
-            pyenv_index = self.path_order.index(last_pyenv)
+            path_index = self.path_order.index(last_instance)
         except ValueError:
             return
+        return path_index
+
+    def _slice_in_paths(self, start_idx, paths):
+        before_path = self.path_order[: start_idx + 1]
+        after_path = self.path_order[start_idx + 2 :]
+        self.path_order = (
+            before_path + [p.as_posix() for p in paths] + after_path
+        )
+
+    def _remove_path(self, path):
+        path_copy = reversed(self.path_order[:])
+        new_order = []
+        target = os.path.normcase(os.path.normpath(os.path.abspath(path)))
+        path_map = {
+            os.path.normcase(os.path.normpath(os.path.abspath(pth))): pth
+            for pth in self.paths.keys()
+        }
+        if target in path_map:
+            del self.paths[path_map.get(target)]
+        for current_path in path_copy:
+            normalized = os.path.normcase(os.path.normpath(os.path.abspath(current_path)))
+            if normalized != target:
+                new_order.append(normalized)
+        new_order = reversed(new_order)
+        self.path_order = new_order
+
+    def _setup_asdf(self):
+        from .asdf import AsdfFinder
+        asdf_index = self._get_last_instance(ASDF_DATA_DIR)
+        self.asdf_finder = AsdfFinder.create(root=ASDF_DATA_DIR, ignore_unsupported=True)
+        root_paths = [p for p in self.asdf_finder.roots]
+        self._slice_in_paths(asdf_index, root_paths)
+        self.paths.update(self.asdf_finder.roots)
+        self._register_finder("asdf", self.asdf_finder)
+
+    def _setup_pyenv(self):
+        from .pyenv import PyenvFinder
+
+        pyenv_index = self._get_last_instance(PYENV_ROOT)
         self.pyenv_finder = PyenvFinder.create(
             root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported
         )
         root_paths = [p for p in self.pyenv_finder.roots]
-        before_path = self.path_order[: pyenv_index + 1]
-        after_path = self.path_order[pyenv_index + 2 :]
-        self.path_order = (
-            before_path + [p.as_posix() for p in root_paths] + after_path
-        )
-        pyenv_shim_path = os.path.join(PYENV_ROOT, "shims")
-        if pyenv_shim_path in self.path_order:
-            self.path_order.remove(pyenv_shim_path)
+        self._slice_in_paths(pyenv_index, root_paths)
+
         self.paths.update(self.pyenv_finder.roots)
-        if pyenv_shim_path in self.paths:
-            del self.paths[pyenv_shim_path]
+        self._remove_path(os.path.join(PYENV_ROOT, "shims"))
         self._register_finder("pyenv", self.pyenv_finder)
 
     def _setup_windows(self):
@@ -396,7 +429,7 @@ class SystemPath(object):
         )
 
 
-@attr.s
+@attr.s(slots=True)
 class PathEntry(BasePath):
     path = attr.ib(default=None, validator=optional_instance_of(Path))
     _children = attr.ib(default=attr.Factory(dict))

pipenv/vendor/pythonfinder/models/pyenv.py

@@ -26,7 +26,7 @@ from .python import PythonVersion
 logger = logging.getLogger(__name__)
 
 
-@attr.s
+@attr.s(slots=True)
 class PyenvFinder(BaseFinder, BasePath):
     root = attr.ib(default=None, validator=optional_instance_of(Path))
     #: ignore_unsupported should come before versions, because its value is used
@@ -34,6 +34,7 @@ class PyenvFinder(BaseFinder, BasePath):
     ignore_unsupported = attr.ib(default=True)
     paths = attr.ib(default=attr.Factory(list))
     roots = attr.ib(default=attr.Factory(defaultdict))
+    version_root = attr.ib(default="versions/*")
     versions = attr.ib()
     pythons = attr.ib()
 
@@ -50,7 +51,7 @@ class PyenvFinder(BaseFinder, BasePath):
             version_order_lines = version_order_file.read_text(encoding="utf-8").splitlines()
 
         version_paths = [
-            p for p in self.root.glob("versions/*")
+            p for p in self.root.glob(self.version_root)
             if not (p.parent.name == "envs" or p.name == "envs")
         ]
         versions = {v.name: v for v in version_paths}
@@ -74,7 +75,7 @@ class PyenvFinder(BaseFinder, BasePath):
     @versions.default
     def get_versions(self):
         versions = defaultdict()
-        bin_ = sysconfig._INSTALL_SCHEMES['posix_prefix']["scripts"]
+        bin_ = "{base}/bin"
         for p in self.get_version_order():
             bin_dir = Path(bin_.format(base=p.as_posix()))
             version_path = None

pipenv/vendor/pythonfinder/models/python.py

@@ -21,7 +21,7 @@ from ..utils import (
 )
 
 
-@attr.s
+@attr.s(slots=True)
 class PythonVersion(object):
     major = attr.ib(default=0)
     minor = attr.ib(default=None)

pipenv/vendor/pythonfinder/utils.py

@@ -54,7 +54,7 @@ def get_python_version(path):
     version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"]
     try:
         c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True,
-                                combine_stderr=False)
+                            combine_stderr=False)
     except OSError:
         raise InvalidPythonVersion("%s is not a valid python path" % path)
     if not c.out:
@@ -92,7 +92,7 @@ def looks_like_python(name):
 
 @lru_cache(maxsize=1024)
 def path_is_python(path):
-    return path_is_executable(path) and looks_like_python(path.name)
+    return path_is_known_executable(path) and looks_like_python(path.name)
 
 
 @lru_cache(maxsize=1024)
@@ -117,7 +117,7 @@ def _filter_none(k, v):
     return False
 
 
-@lru_cache(maxsize=128)
+@lru_cache(maxsize=1024)
 def filter_pythons(path):
     """Return all valid pythons in a given path"""
     if not isinstance(path, vistir.compat.Path):