Update pythonfinder

Signed-off-by: Dan Ryan <dan@danryan.co>

pipenv/vendor/pythonfinder/models/__init__.py

@@ -3,6 +3,7 @@ from __future__ import print_function, absolute_import
 import abc
 import operator
 import six
+from itertools import chain
 from ..utils import KNOWN_EXTS
 
 
@@ -42,28 +43,52 @@ class BasePath(object):
         found = next((children[(self.path / child).as_posix()] for child in valid_names if (self.path / child).as_posix() in children), None)
         return found
 
-    def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None):
+    def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None):
+        """Search for a specific python version on the path. Return all copies
+
+        :param major: Major python version to search for.
+        :type major: int
+        :param int minor: Minor python version to search for, defaults to None
+        :param int patch: Patch python version to search for, defaults to None
+        :param bool pre: Search for prereleases (default None) - prioritize releases if None
+        :param bool dev: Search for devreleases (default None) - prioritize releases if None
+        :param str arch: Architecture to include, e.g. '64bit', defaults to None
+        :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested.
+        :rtype: List[:class:`~pythonfinder.models.PathEntry`]
+        """
+
+        sub_finder = operator.methodcaller(
+            "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch
+        )
+        if not self.is_dir:
+            return sub_finder(self)
+        path_filter = filter(None, (sub_finder(p) for p in self.children.values()))
+        version_sort = operator.attrgetter("as_python.version_sort")
+        return [c for c in sorted(path_filter, key=version_sort, reverse=True)]
+
+    def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None):
         """Search or self for the specified Python version and return the first match.
 
         :param major: Major version number.
         :type major: int
-        :param minor: Minor python version, defaults to None
-        :param minor: int, optional
-        :param patch: Patch python version, defaults to None
-        :param patch: int, optional
+        :param int minor: Minor python version to search for, defaults to None
+        :param int patch: Patch python version to search for, defaults to None
+        :param bool pre: Search for prereleases (default None) - prioritize releases if None
+        :param bool dev: Search for devreleases (default None) - prioritize releases if None
+        :param str arch: Architecture to include, e.g. '64bit', defaults to None
         :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested.
         """
 
         version_matcher = operator.methodcaller(
-            "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev
+            "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch
         )
         is_py = operator.attrgetter("is_python")
         py_version = operator.attrgetter("as_python")
         if not self.is_dir:
-            if self.is_python and self.as_python.matches(major=major, minor=minor, patch=patch, pre=pre, dev=dev):
+            if self.is_python and version_matcher(self.as_python):
                 return self
             return
-        finder = ((child, child.as_python) for child in self.children.values() if child.is_python and child.as_python)
+        finder = ((child, child.as_python) for child in chain(*filter(None, self.pythons.values())) if child.as_python)
         py_filter = filter(
             None, filter(lambda child: version_matcher(child[1]), finder)
         )

pipenv/vendor/pythonfinder/models/path.py

@@ -1,10 +1,12 @@
 # -*- coding=utf-8 -*-
 from __future__ import print_function, absolute_import
 import attr
+import copy
 import operator
 import os
 import sys
 from collections import defaultdict
+from itertools import chain
 from . import BasePath
 from .python import PythonVersion
 from ..environment import PYENV_INSTALLED, PYENV_ROOT
@@ -13,7 +15,7 @@ from ..utils import (
     optional_instance_of,
     filter_pythons,
     path_is_known_executable,
-    is_python_name,
+    looks_like_python,
     ensure_path,
     fs_str
 )
@@ -31,32 +33,49 @@ class SystemPath(object):
     _executables = attr.ib(default=attr.Factory(list))
     _python_executables = attr.ib(default=attr.Factory(list))
     path_order = attr.ib(default=attr.Factory(list))
-    python_version_dict = attr.ib()
+    python_version_dict = attr.ib(default=attr.Factory(defaultdict))
     only_python = attr.ib(default=False)
     pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath"))
     system = attr.ib(default=False)
 
+    __finders = attr.ib(default=attr.Factory(list))
+
+    def _register_finder(self, finder):
+        if not finder in self.__finders:
+            self.__finders.append(finder)
+
     @property
     def executables(self):
         if not self._executables:
-            self._executables = [p for p in self.paths.values() if p.is_executable]
+            self._executables = [p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable]
         return self._executables
 
     @property
     def python_executables(self):
+        python_executables = {}
         if not self._python_executables:
-            self._python_executables = [p for p in self.paths.values() if p.is_python]
+            for child in self.paths.values():
+                if child.pythons:
+                    python_executables.update(dict(child.pythons))
+            for finder in self.__finders:
+                if finder.pythons:
+                    python_executables.update(dict(finder.pythons))
+            self._python_executables = python_executables
         return self._python_executables
 
-    @python_version_dict.default
     def get_python_version_dict(self):
         version_dict = defaultdict(list)
-        for p in self.python_executables:
-            try:
-                version_object = PythonVersion.from_path(p)
-            except (ValueError, InvalidPythonVersion):
+        for finder in self.__finders:
+            for version, entry in finder.versions.items():
+                if entry not in version_dict[version]:
+                    version_dict[version].append(entry)
+        for p, entry in self.python_executables.items():
+            version = entry.as_python
+            if not version:
                 continue
-            version_dict[version_object.version_tuple].append(version_object)
+            version = version.version_tuple
+            if version and entry not in version_dict[version]:
+                version_dict[version].append(entry)
         return version_dict
 
     def __attrs_post_init__(self):
@@ -73,7 +92,7 @@ class SystemPath(object):
         else:
             bin_dir = 'bin'
         if venv and (self.system or self.global_search):
-            p = Path(venv)
+            p = ensure_path(venv)
             self.path_order = [(p / bin_dir).as_posix()] + self.path_order
             self.paths[p] = PathEntry.create(
                 path=p, is_root=True, only_python=False
@@ -84,9 +103,10 @@ class SystemPath(object):
             if syspath_bin.name != bin_dir and syspath_bin.joinpath(bin_dir).exists():
                 syspath_bin = syspath_bin / bin_dir
             self.path_order = [syspath_bin.as_posix()] + self.path_order
-            self.paths[syspath_bin.as_posix()] = PathEntry.create(
+            self.paths[syspath_bin] = PathEntry.create(
                 path=syspath_bin, is_root=True, only_python=False
             )
+        self.python_version_dict = self.get_python_version_dict()
 
     def _setup_pyenv(self):
         from .pyenv import PyenvFinder
@@ -110,6 +130,7 @@ class SystemPath(object):
             before_path + [p.path.as_posix() for p in root_paths] + after_path
         )
         self.paths.update({p.path: p for p in root_paths})
+        self._register_finder(self.pyenv_finder)
 
     def _setup_windows(self):
         from .windows import WindowsFinder
@@ -119,15 +140,17 @@ class SystemPath(object):
         path_addition = [p.path.as_posix() for p in root_paths]
         self.path_order = self.path_order[:] + path_addition
         self.paths.update({p.path: p for p in root_paths})
+        self._register_finder(self.windows_finder)
 
     def get_path(self, path):
-        path = Path(path)
+        path = ensure_path(path)
         _path = self.paths.get(path.as_posix())
         if not _path and path.as_posix() in self.path_order:
-            self.paths[path.as_posix()] = PathEntry.create(
-                path=path.resolve(), is_root=True, only_python=self.only_python
+            _path =  PathEntry.create(
+                path=path.absolute(), is_root=True, only_python=self.only_python
             )
-        return self.paths.get(path.as_posix())
+            self.paths[path.as_posix()] = _path
+        return _path
 
     def find_all(self, executable):
         """Search the path for an executable. Return all copies.
@@ -151,21 +174,22 @@ class SystemPath(object):
         filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order))
         return next((f for f in filtered), None)
 
-    def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None):
+    def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None):
         """Search for a specific python version on the path. Return all copies
 
         :param major: Major python version to search for.
         :type major: int
-        :param minor: Minor python version to search for, defaults to None
-        :param minor: int, optional
-        :param path: Patch python version to search for, defaults to None
-        :param path: int, optional
+        :param int minor: Minor python version to search for, defaults to None
+        :param int patch: Patch python version to search for, defaults to None
+        :param bool pre: Search for prereleases (default None) - prioritize releases if None
+        :param bool dev: Search for devreleases (default None) - prioritize releases if None
+        :param str arch: Architecture to include, e.g. '64bit', defaults to None
         :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested.
         :rtype: List[:class:`~pythonfinder.models.PathEntry`]
         """
 
         sub_finder = operator.methodcaller(
-            "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev
+            "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch
         )
         if os.name == "nt" and self.windows_finder:
             windows_finder_version = sub_finder(self.windows_finder)
@@ -176,22 +200,33 @@ class SystemPath(object):
         version_sort = operator.attrgetter("as_python.version_sort")
         return [c for c in sorted(path_filter, key=version_sort, reverse=True)]
 
-    def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None):
+    def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None):
         """Search for a specific python version on the path.
 
         :param major: Major python version to search for.
         :type major: int
-        :param minor: Minor python version to search for, defaults to None
-        :param minor: int, optional
-        :param path: Patch python version to search for, defaults to None
-        :param path: int, optional
+        :param int minor: Minor python version to search for, defaults to None
+        :param int patch: Patch python version to search for, defaults to None
+        :param bool pre: Search for prereleases (default None) - prioritize releases if None
+        :param bool dev: Search for devreleases (default None) - prioritize releases if None
+        :param str arch: Architecture to include, e.g. '64bit', defaults to None
         :return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested.
         :rtype: :class:`~pythonfinder.models.PathEntry`
         """
 
         sub_finder = operator.methodcaller(
-            "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev
+            "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch
         )
+        if major and minor and patch:
+            _tuple_pre = pre if pre is not None else False
+            _tuple_dev = dev if dev is not None else False
+            version_tuple = (major, minor_, patch, _tuple_pre, _tuple_dev)
+            version_tuple_pre = (major, minor, patch, True, False)
+            version = self.python_version_dict.get(version_tuple)
+            if not version:
+                version = self.python_version_dict.get(version_tuple_pre)
+            if version:
+                return first(version.comes_from)
         if os.name == "nt" and self.windows_finder:
             windows_finder_version = sub_finder(self.windows_finder)
             if windows_finder_version:
@@ -243,7 +278,7 @@ class PathEntry(BasePath):
     is_root = attr.ib(default=True)
     only_python = attr.ib(default=False)
     py_version = attr.ib(default=None)
-    pythons = attr.ib(default=None)
+    pythons = attr.ib()
 
     def __str__(self):
         return fs_str('{0}'.format(self.path.as_posix()))
@@ -259,11 +294,27 @@ class PathEntry(BasePath):
     def children(self):
         if not self._children and self.is_dir and self.is_root:
             self._children = {
-                child.as_posix(): PathEntry(path=child, is_root=False)
+                child.as_posix(): PathEntry.create(path=child, is_root=False)
                 for child in self._filter_children()
             }
+        elif not self.is_dir:
+            return {self.path.as_posix(): self}
         return self._children
 
+    @pythons.default
+    def get_pythons(self):
+        pythons = defaultdict()
+        if self.is_dir:
+            for path, entry in self.children.items():
+                _path = ensure_path(entry.path)
+                if entry.is_python:
+                    pythons[_path.as_posix()] = entry
+        else:
+            if self.is_python:
+                _path = ensure_path(self.path)
+                pythons[_path.as_posix()] = copy.deepcopy(self)
+        return pythons
+
     @property
     def as_python(self):
         if not self.is_dir and self.is_python:
@@ -292,9 +343,14 @@ class PathEntry(BasePath):
         """
 
         target = ensure_path(path)
-        _new = cls(
-            path=target, is_root=is_root, only_python=only_python, pythons=pythons
-        )
+        creation_args = {
+            "path": target,
+            "is_root": is_root,
+            "only_python": only_python
+        }
+        if pythons:
+            creation_args["pythons"] = pythons
+        _new = cls(**creation_args)
         if pythons and only_python:
             children = {}
             for pth, python in pythons.items():
@@ -311,7 +367,11 @@ class PathEntry(BasePath):
 
     @property
     def is_dir(self):
-        return self.path.is_dir()
+        try:
+            ret_val = self.path.is_dir()
+        except OSError:
+            ret_val = False
+        return ret_val
 
     @property
     def is_executable(self):
@@ -320,7 +380,7 @@ class PathEntry(BasePath):
     @property
     def is_python(self):
         return self.is_executable and (
-            self.py_version or is_python_name(self.path.name)
+            self.py_version or looks_like_python(self.path.name)
         )
 
 

pipenv/vendor/pythonfinder/models/pyenv.py

@@ -18,6 +18,7 @@ except ImportError:
 class PyenvFinder(BaseFinder):
     root = attr.ib(default=None, validator=optional_instance_of(Path))
     versions = attr.ib()
+    pythons = attr.ib()
 
     @versions.default
     def get_versions(self):
@@ -34,6 +35,21 @@ class PyenvFinder(BaseFinder):
             versions[version_tuple] = VersionPath.create(path=p.resolve(), only_python=True)
         return versions
 
+    @pythons.default
+    def get_pythons(self):
+        pythons = defaultdict()
+        for v in self.versions.values():
+            for p in v.paths.values():
+                _path = p.path
+                try:
+                    _path = _path.resolve()
+                except OSError:
+                    _path = _path.absolute()
+                _path = _path.as_posix()
+                if p.is_python:
+                    pythons[_path] = p
+        return pythons
+
     @classmethod
     def create(cls, root):
         root = ensure_path(root)

pipenv/vendor/pythonfinder/models/python.py

@@ -2,6 +2,7 @@
 from __future__ import print_function, absolute_import
 import attr
 import copy
+from collections import defaultdict
 import platform
 from packaging.version import parse as parse_version, Version
 from ..environment import SYSTEM_ARCH
@@ -67,13 +68,14 @@ class PythonVersion(object):
             self.is_devrelease,
         )
 
-    def matches(self, major=None, minor=None, patch=None, pre=False, dev=False):
+    def matches(self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None):
         return (
             (major is None or self.major == major)
             and (minor is None or self.minor == minor)
             and (patch is None or self.patch == patch)
             and (pre is None or self.is_prerelease == pre)
             and (dev is None or self.is_devrelease == dev)
+            and (arch is None or self.architecture == arch)
         )
 
     def as_major(self):
@@ -143,7 +145,7 @@ class PythonVersion(object):
         from .path import PathEntry
 
         if not isinstance(path, PathEntry):
-            path = PathEntry(path)
+            path = PathEntry.create(path, is_root=False, only_python=True)
         if not path.is_python:
             raise ValueError("Not a valid python path: %s" % path.path)
             return
@@ -192,3 +194,26 @@ class PythonVersion(object):
     @classmethod
     def create(cls, **kwargs):
         return cls(**kwargs)
+
+
+@attr.s
+class VersionMap(object):
+    versions = attr.ib(default=attr.Factory(defaultdict(list)))
+
+    def add_entry(self, entry):
+        version = entry.as_python
+        if version:
+            entries = versions[version.version_tuple]
+            paths = {p.path for p in self.versions.get(version.version_tuple, [])}
+            if entry.path not in paths:
+                self.versions[version.version_tuple].append(entry)
+
+    def merge(self, target):
+        for version, entries in target.versions.items():
+            if version not in self.versions:
+                self.versions[version] = entries
+            else:
+                current_entries = {p.path for p in self.versions.get(version)}
+                new_entries = {p.path for p in entries}
+                new_entries -= current_entries
+                self.versions[version].append([e for e in entries if e.path in new_entries])

pipenv/vendor/pythonfinder/models/windows.py

@@ -5,7 +5,7 @@ import operator
 from collections import defaultdict
 from . import BaseFinder
 from .path import PathEntry
-from .python import PythonVersion
+from .python import PythonVersion, VersionMap
 from ..exceptions import InvalidPythonVersion
 from ..utils import ensure_path
 
@@ -15,10 +15,11 @@ class WindowsFinder(BaseFinder):
     paths = attr.ib(default=attr.Factory(list))
     version_list = attr.ib(default=attr.Factory(list))
     versions = attr.ib()
+    pythons = attr.ib()
 
-    def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None):
+    def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None):
         version_matcher = operator.methodcaller(
-            "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev
+            "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None
         )
         py_filter = filter(
             None, filter(lambda c: version_matcher(c), self.version_list)
@@ -26,10 +27,10 @@ class WindowsFinder(BaseFinder):
         version_sort = operator.attrgetter("version_sort")
         return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)]
 
-    def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None):
+    def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None):
         return next((
             v for v in self.find_all_python_versions(
-                major=major, minor=minor, patch=patch, pre=pre, dev=dev
+                major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None
             )), None
         )
 
@@ -57,6 +58,14 @@ class WindowsFinder(BaseFinder):
             self.paths.append(base_dir)
         return versions
 
+    @pythons.default
+    def get_pythons(self):
+        pythons = defaultdict()
+        for version in self.version_list:
+            _path = ensure_path(version.comes_from.path)
+            pythons[_path.as_posix()] = version.comes_from
+        return pythons
+
     @classmethod
     def create(cls):
         return cls()

pipenv/vendor/pythonfinder/pythonfinder.py

@@ -50,15 +50,20 @@ class Finder(object):
     def which(self, exe):
         return self.system_path.which(exe)
 
-    def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None):
+    def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, arch=None):
         from .models import PythonVersion
         if isinstance(major, six.string_types) and pre is None and minor is None and dev is None and patch is None:
+            if arch is None and '-' in major:
+                major, arch = major.rsplit('-', 1)
+                if not arch.isnumeric():
+                    major = "{0}-{1}".format(major, arch)
             version_dict = PythonVersion.parse(major)
             major = version_dict.get("major", major)
             minor = version_dict.get("minor", minor)
             patch = version_dict.get("patch", patch)
-            pre = version_dict.get("is_prerelease", pre) if pre is not None else pre
-            dev = version_dict.get("is_devrelease", dev) if dev is not None else dev
+            pre = version_dict.get("is_prerelease", pre) if pre is None else pre
+            dev = version_dict.get("is_devrelease", dev) if dev is None else dev
+            arch = version_dict.get("architecture", arch) if arch is None else arch
         if os.name == "nt":
             match = self.windows_finder.find_python_version(
                 major, minor=minor, patch=patch, pre=pre, dev=dev
@@ -66,15 +71,24 @@ class Finder(object):
             if match:
                 return match
         return self.system_path.find_python_version(
-            major=major, minor=minor, patch=patch, pre=pre, dev=dev
+            major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch
         )
 
-    def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None):
+    def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None):
         version_sort = operator.attrgetter("as_python.version_sort")
-        versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev)
+        versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch)
         if not isinstance(versions, list):
             versions = [versions,]
         if os.name == 'nt':
-            windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev)
-            versions = versions + list(windows_versions)
-        return sorted(versions, key=version_sort, reverse=True)
+            windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch)
+            versions = list(windows_versions) + versions
+        paths = sorted(versions, key=version_sort, reverse=True)
+        path_map = {}
+        for path in paths:
+            try:
+                resolved_path = path.path.resolve()
+            except OSError:
+                resolved_path = path.path.absolute()
+            if not path_map.get(resolved_path.as_posix()):
+                path_map[resolved_path.as_posix()] = path
+        return list(path_map.values())

pipenv/vendor/pythonfinder/utils.py

@@ -72,7 +72,7 @@ def path_is_known_executable(path):
     )
 
 
-def is_python_name(name):
+def looks_like_python(name):
     rules = ["*python", "*python?", "*python?.?", "*python?.?m"]
     match_rules = []
     for rule in rules:
@@ -88,7 +88,7 @@ def is_python_name(name):
 
 
 def path_is_python(path):
-    return path_is_executable(path) and is_python_name(path.name)
+    return path_is_executable(path) and looks_like_python(path.name)
 
 
 def ensure_path(path):
@@ -101,8 +101,13 @@ def ensure_path(path):
     """
 
     if isinstance(path, Path):
-        return Path(os.path.expandvars(path.as_posix()))
-    return Path(os.path.expandvars(path))
+        path = path.as_posix()
+    path = Path(os.path.expandvars(path))
+    try:
+        path = path.resolve()
+    except OSError:
+        path = path.absolute()
+    return path
 
 
 def _filter_none(k, v):