From a709bf845f4d0422d6dd0b682aca4c702d039680 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 02:37:32 -0400 Subject: [PATCH] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/__init__.py | 41 ++++-- pipenv/vendor/pythonfinder/models/path.py | 130 +++++++++++++----- pipenv/vendor/pythonfinder/models/pyenv.py | 16 +++ pipenv/vendor/pythonfinder/models/python.py | 29 +++- pipenv/vendor/pythonfinder/models/windows.py | 19 ++- pipenv/vendor/pythonfinder/pythonfinder.py | 32 +++-- pipenv/vendor/pythonfinder/utils.py | 13 +- 7 files changed, 217 insertions(+), 63 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index 0c6f0134..b1b56d43 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -3,6 +3,7 @@ from __future__ import print_function, absolute_import import abc import operator import six +from itertools import chain from ..utils import KNOWN_EXTS @@ -42,28 +43,52 @@ class BasePath(object): found = next((children[(self.path / child).as_posix()] for child in valid_names if (self.path / child).as_posix() in children), None) return found - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + """Search for a specific python version on the path. Return all copies + + :param major: Major python version to search for. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. + :rtype: List[:class:`~pythonfinder.models.PathEntry`] + """ + + sub_finder = operator.methodcaller( + "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + ) + if not self.is_dir: + return sub_finder(self) + path_filter = filter(None, (sub_finder(p) for p in self.children.values())) + version_sort = operator.attrgetter("as_python.version_sort") + return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): """Search or self for the specified Python version and return the first match. :param major: Major version number. :type major: int - :param minor: Minor python version, defaults to None - :param minor: int, optional - :param patch: Patch python version, defaults to None - :param patch: int, optional + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") if not self.is_dir: - if self.is_python and self.as_python.matches(major=major, minor=minor, patch=patch, pre=pre, dev=dev): + if self.is_python and version_matcher(self.as_python): return self return - finder = ((child, child.as_python) for child in self.children.values() if child.is_python and child.as_python) + finder = ((child, child.as_python) for child in chain(*filter(None, self.pythons.values())) if child.as_python) py_filter = filter( None, filter(lambda child: version_matcher(child[1]), finder) ) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 893b6af0..5e1807ab 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -1,10 +1,12 @@ # -*- coding=utf-8 -*- from __future__ import print_function, absolute_import import attr +import copy import operator import os import sys from collections import defaultdict +from itertools import chain from . import BasePath from .python import PythonVersion from ..environment import PYENV_INSTALLED, PYENV_ROOT @@ -13,7 +15,7 @@ from ..utils import ( optional_instance_of, filter_pythons, path_is_known_executable, - is_python_name, + looks_like_python, ensure_path, fs_str ) @@ -31,32 +33,49 @@ class SystemPath(object): _executables = attr.ib(default=attr.Factory(list)) _python_executables = attr.ib(default=attr.Factory(list)) path_order = attr.ib(default=attr.Factory(list)) - python_version_dict = attr.ib() + python_version_dict = attr.ib(default=attr.Factory(defaultdict)) only_python = attr.ib(default=False) pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) system = attr.ib(default=False) + __finders = attr.ib(default=attr.Factory(list)) + + def _register_finder(self, finder): + if not finder in self.__finders: + self.__finders.append(finder) + @property def executables(self): if not self._executables: - self._executables = [p for p in self.paths.values() if p.is_executable] + self._executables = [p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable] return self._executables @property def python_executables(self): + python_executables = {} if not self._python_executables: - self._python_executables = [p for p in self.paths.values() if p.is_python] + for child in self.paths.values(): + if child.pythons: + python_executables.update(dict(child.pythons)) + for finder in self.__finders: + if finder.pythons: + python_executables.update(dict(finder.pythons)) + self._python_executables = python_executables return self._python_executables - @python_version_dict.default def get_python_version_dict(self): version_dict = defaultdict(list) - for p in self.python_executables: - try: - version_object = PythonVersion.from_path(p) - except (ValueError, InvalidPythonVersion): + for finder in self.__finders: + for version, entry in finder.versions.items(): + if entry not in version_dict[version]: + version_dict[version].append(entry) + for p, entry in self.python_executables.items(): + version = entry.as_python + if not version: continue - version_dict[version_object.version_tuple].append(version_object) + version = version.version_tuple + if version and entry not in version_dict[version]: + version_dict[version].append(entry) return version_dict def __attrs_post_init__(self): @@ -73,7 +92,7 @@ class SystemPath(object): else: bin_dir = 'bin' if venv and (self.system or self.global_search): - p = Path(venv) + p = ensure_path(venv) self.path_order = [(p / bin_dir).as_posix()] + self.path_order self.paths[p] = PathEntry.create( path=p, is_root=True, only_python=False @@ -84,9 +103,10 @@ class SystemPath(object): if syspath_bin.name != bin_dir and syspath_bin.joinpath(bin_dir).exists(): syspath_bin = syspath_bin / bin_dir self.path_order = [syspath_bin.as_posix()] + self.path_order - self.paths[syspath_bin.as_posix()] = PathEntry.create( + self.paths[syspath_bin] = PathEntry.create( path=syspath_bin, is_root=True, only_python=False ) + self.python_version_dict = self.get_python_version_dict() def _setup_pyenv(self): from .pyenv import PyenvFinder @@ -110,6 +130,7 @@ class SystemPath(object): before_path + [p.path.as_posix() for p in root_paths] + after_path ) self.paths.update({p.path: p for p in root_paths}) + self._register_finder(self.pyenv_finder) def _setup_windows(self): from .windows import WindowsFinder @@ -119,15 +140,17 @@ class SystemPath(object): path_addition = [p.path.as_posix() for p in root_paths] self.path_order = self.path_order[:] + path_addition self.paths.update({p.path: p for p in root_paths}) + self._register_finder(self.windows_finder) def get_path(self, path): - path = Path(path) + path = ensure_path(path) _path = self.paths.get(path.as_posix()) if not _path and path.as_posix() in self.path_order: - self.paths[path.as_posix()] = PathEntry.create( - path=path.resolve(), is_root=True, only_python=self.only_python + _path = PathEntry.create( + path=path.absolute(), is_root=True, only_python=self.only_python ) - return self.paths.get(path.as_posix()) + self.paths[path.as_posix()] = _path + return _path def find_all(self, executable): """Search the path for an executable. Return all copies. @@ -151,21 +174,22 @@ class SystemPath(object): filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) return next((f for f in filtered), None) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. :type major: int - :param minor: Minor python version to search for, defaults to None - :param minor: int, optional - :param path: Patch python version to search for, defaults to None - :param path: int, optional + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev + "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) @@ -176,22 +200,33 @@ class SystemPath(object): version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): """Search for a specific python version on the path. :param major: Major python version to search for. :type major: int - :param minor: Minor python version to search for, defaults to None - :param minor: int, optional - :param path: Patch python version to search for, defaults to None - :param path: int, optional + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None :return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. :rtype: :class:`~pythonfinder.models.PathEntry` """ sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev + "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) + if major and minor and patch: + _tuple_pre = pre if pre is not None else False + _tuple_dev = dev if dev is not None else False + version_tuple = (major, minor_, patch, _tuple_pre, _tuple_dev) + version_tuple_pre = (major, minor, patch, True, False) + version = self.python_version_dict.get(version_tuple) + if not version: + version = self.python_version_dict.get(version_tuple_pre) + if version: + return first(version.comes_from) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: @@ -243,7 +278,7 @@ class PathEntry(BasePath): is_root = attr.ib(default=True) only_python = attr.ib(default=False) py_version = attr.ib(default=None) - pythons = attr.ib(default=None) + pythons = attr.ib() def __str__(self): return fs_str('{0}'.format(self.path.as_posix())) @@ -259,11 +294,27 @@ class PathEntry(BasePath): def children(self): if not self._children and self.is_dir and self.is_root: self._children = { - child.as_posix(): PathEntry(path=child, is_root=False) + child.as_posix(): PathEntry.create(path=child, is_root=False) for child in self._filter_children() } + elif not self.is_dir: + return {self.path.as_posix(): self} return self._children + @pythons.default + def get_pythons(self): + pythons = defaultdict() + if self.is_dir: + for path, entry in self.children.items(): + _path = ensure_path(entry.path) + if entry.is_python: + pythons[_path.as_posix()] = entry + else: + if self.is_python: + _path = ensure_path(self.path) + pythons[_path.as_posix()] = copy.deepcopy(self) + return pythons + @property def as_python(self): if not self.is_dir and self.is_python: @@ -292,9 +343,14 @@ class PathEntry(BasePath): """ target = ensure_path(path) - _new = cls( - path=target, is_root=is_root, only_python=only_python, pythons=pythons - ) + creation_args = { + "path": target, + "is_root": is_root, + "only_python": only_python + } + if pythons: + creation_args["pythons"] = pythons + _new = cls(**creation_args) if pythons and only_python: children = {} for pth, python in pythons.items(): @@ -311,7 +367,11 @@ class PathEntry(BasePath): @property def is_dir(self): - return self.path.is_dir() + try: + ret_val = self.path.is_dir() + except OSError: + ret_val = False + return ret_val @property def is_executable(self): @@ -320,7 +380,7 @@ class PathEntry(BasePath): @property def is_python(self): return self.is_executable and ( - self.py_version or is_python_name(self.path.name) + self.py_version or looks_like_python(self.path.name) ) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 8545ac59..e61e7153 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -18,6 +18,7 @@ except ImportError: class PyenvFinder(BaseFinder): root = attr.ib(default=None, validator=optional_instance_of(Path)) versions = attr.ib() + pythons = attr.ib() @versions.default def get_versions(self): @@ -34,6 +35,21 @@ class PyenvFinder(BaseFinder): versions[version_tuple] = VersionPath.create(path=p.resolve(), only_python=True) return versions + @pythons.default + def get_pythons(self): + pythons = defaultdict() + for v in self.versions.values(): + for p in v.paths.values(): + _path = p.path + try: + _path = _path.resolve() + except OSError: + _path = _path.absolute() + _path = _path.as_posix() + if p.is_python: + pythons[_path] = p + return pythons + @classmethod def create(cls, root): root = ensure_path(root) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 4ff364fa..687f1d43 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -2,6 +2,7 @@ from __future__ import print_function, absolute_import import attr import copy +from collections import defaultdict import platform from packaging.version import parse as parse_version, Version from ..environment import SYSTEM_ARCH @@ -67,13 +68,14 @@ class PythonVersion(object): self.is_devrelease, ) - def matches(self, major=None, minor=None, patch=None, pre=False, dev=False): + def matches(self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None): return ( (major is None or self.major == major) and (minor is None or self.minor == minor) and (patch is None or self.patch == patch) and (pre is None or self.is_prerelease == pre) and (dev is None or self.is_devrelease == dev) + and (arch is None or self.architecture == arch) ) def as_major(self): @@ -143,7 +145,7 @@ class PythonVersion(object): from .path import PathEntry if not isinstance(path, PathEntry): - path = PathEntry(path) + path = PathEntry.create(path, is_root=False, only_python=True) if not path.is_python: raise ValueError("Not a valid python path: %s" % path.path) return @@ -192,3 +194,26 @@ class PythonVersion(object): @classmethod def create(cls, **kwargs): return cls(**kwargs) + + +@attr.s +class VersionMap(object): + versions = attr.ib(default=attr.Factory(defaultdict(list))) + + def add_entry(self, entry): + version = entry.as_python + if version: + entries = versions[version.version_tuple] + paths = {p.path for p in self.versions.get(version.version_tuple, [])} + if entry.path not in paths: + self.versions[version.version_tuple].append(entry) + + def merge(self, target): + for version, entries in target.versions.items(): + if version not in self.versions: + self.versions[version] = entries + else: + current_entries = {p.path for p in self.versions.get(version)} + new_entries = {p.path for p in entries} + new_entries -= current_entries + self.versions[version].append([e for e in entries if e.path in new_entries]) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index f33a4807..90f2d803 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -5,7 +5,7 @@ import operator from collections import defaultdict from . import BaseFinder from .path import PathEntry -from .python import PythonVersion +from .python import PythonVersion, VersionMap from ..exceptions import InvalidPythonVersion from ..utils import ensure_path @@ -15,10 +15,11 @@ class WindowsFinder(BaseFinder): paths = attr.ib(default=attr.Factory(list)) version_list = attr.ib(default=attr.Factory(list)) versions = attr.ib() + pythons = attr.ib() - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -26,10 +27,10 @@ class WindowsFinder(BaseFinder): version_sort = operator.attrgetter("version_sort") return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): return next(( v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None )), None ) @@ -57,6 +58,14 @@ class WindowsFinder(BaseFinder): self.paths.append(base_dir) return versions + @pythons.default + def get_pythons(self): + pythons = defaultdict() + for version in self.version_list: + _path = ensure_path(version.comes_from.path) + pythons[_path.as_posix()] = version.comes_from + return pythons + @classmethod def create(cls): return cls() diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 891a3a17..c74eadeb 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -50,15 +50,20 @@ class Finder(object): def which(self, exe): return self.system_path.which(exe) - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, arch=None): from .models import PythonVersion if isinstance(major, six.string_types) and pre is None and minor is None and dev is None and patch is None: + if arch is None and '-' in major: + major, arch = major.rsplit('-', 1) + if not arch.isnumeric(): + major = "{0}-{1}".format(major, arch) version_dict = PythonVersion.parse(major) major = version_dict.get("major", major) minor = version_dict.get("minor", minor) patch = version_dict.get("patch", patch) - pre = version_dict.get("is_prerelease", pre) if pre is not None else pre - dev = version_dict.get("is_devrelease", dev) if dev is not None else dev + pre = version_dict.get("is_prerelease", pre) if pre is None else pre + dev = version_dict.get("is_devrelease", dev) if dev is None else dev + arch = version_dict.get("architecture", arch) if arch is None else arch if os.name == "nt": match = self.windows_finder.find_python_version( major, minor=minor, patch=patch, pre=pre, dev=dev @@ -66,15 +71,24 @@ class Finder(object): if match: return match return self.system_path.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): version_sort = operator.attrgetter("as_python.version_sort") - versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) + versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) if not isinstance(versions, list): versions = [versions,] if os.name == 'nt': - windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) - versions = versions + list(windows_versions) - return sorted(versions, key=version_sort, reverse=True) + windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) + versions = list(windows_versions) + versions + paths = sorted(versions, key=version_sort, reverse=True) + path_map = {} + for path in paths: + try: + resolved_path = path.path.resolve() + except OSError: + resolved_path = path.path.absolute() + if not path_map.get(resolved_path.as_posix()): + path_map[resolved_path.as_posix()] = path + return list(path_map.values()) diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 0cc5370a..df837ea9 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -72,7 +72,7 @@ def path_is_known_executable(path): ) -def is_python_name(name): +def looks_like_python(name): rules = ["*python", "*python?", "*python?.?", "*python?.?m"] match_rules = [] for rule in rules: @@ -88,7 +88,7 @@ def is_python_name(name): def path_is_python(path): - return path_is_executable(path) and is_python_name(path.name) + return path_is_executable(path) and looks_like_python(path.name) def ensure_path(path): @@ -101,8 +101,13 @@ def ensure_path(path): """ if isinstance(path, Path): - return Path(os.path.expandvars(path.as_posix())) - return Path(os.path.expandvars(path)) + path = path.as_posix() + path = Path(os.path.expandvars(path)) + try: + path = path.resolve() + except OSError: + path = path.absolute() + return path def _filter_none(k, v):