From ea1096d9b560a3793c996f08a5ea48d45e6fb534 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 21 Jan 2019 19:41:09 -0500 Subject: [PATCH] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/__init__.py | 4 +- pipenv/vendor/pythonfinder/__main__.py | 7 +- pipenv/vendor/pythonfinder/cli.py | 22 +- pipenv/vendor/pythonfinder/environment.py | 12 +- pipenv/vendor/pythonfinder/exceptions.py | 2 +- pipenv/vendor/pythonfinder/models/mixins.py | 319 +++++++++--- pipenv/vendor/pythonfinder/models/path.py | 455 +++++++++-------- pipenv/vendor/pythonfinder/models/python.py | 505 ++++++++++++------- pipenv/vendor/pythonfinder/models/windows.py | 112 ++-- pipenv/vendor/pythonfinder/pythonfinder.py | 194 +++++-- pipenv/vendor/pythonfinder/utils.py | 89 +++- 11 files changed, 1146 insertions(+), 575 deletions(-) diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 8e12a198..9421573d 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function -__version__ = '1.1.10' +__version__ = '1.1.11' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/__main__.py b/pipenv/vendor/pythonfinder/__main__.py index c804d573..3083e72d 100644 --- a/pipenv/vendor/pythonfinder/__main__.py +++ b/pipenv/vendor/pythonfinder/__main__.py @@ -1,12 +1,17 @@ +#!env python +# -*- coding=utf-8 -*- + from __future__ import absolute_import import os import sys +from pythonfinder.cli import cli + + PYTHONFINDER_MAIN = os.path.dirname(os.path.abspath(__file__)) PYTHONFINDER_PACKAGE = os.path.dirname(PYTHONFINDER_MAIN) -from pythonfinder import cli as cli if __name__ == "__main__": sys.exit(cli()) diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index 221cb2fd..7dc4ebd4 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -1,9 +1,11 @@ -#!/usr/bin/env python # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function, unicode_literals + +import sys + import click import crayons -import sys + from . import __version__ from .pythonfinder import Finder @@ -17,10 +19,18 @@ from .pythonfinder import Finder @click.option( "--version", is_flag=True, default=False, help="Display PythonFinder version." ) -@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", help="Ignore unsupported python versions.") -@click.version_option(prog_name='pyfinder', version=__version__) +@click.option( + "--ignore-unsupported/--no-unsupported", + is_flag=True, + default=True, + envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", + help="Ignore unsupported python versions.", +) +@click.version_option(prog_name="pyfinder", version=__version__) @click.pass_context -def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True): +def cli( + ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True +): if version: click.echo( "{0} version {1}".format( diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index e8878403..ce21fb79 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -1,5 +1,6 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function + import os import platform import sys @@ -34,3 +35,12 @@ else: IGNORE_UNSUPPORTED = bool(os.environ.get("PYTHONFINDER_IGNORE_UNSUPPORTED", False)) MYPY_RUNNING = os.environ.get("MYPY_RUNNING", is_type_checking()) + +def get_shim_paths(): + shim_paths = [] + if ASDF_INSTALLED: + shim_paths.append(os.path.join(ASDF_DATA_DIR, "shims")) + if PYENV_INSTALLED: + shim_paths.append(os.path.join(PYENV_ROOT, "shims")) + return [os.path.normpath(os.path.normcase(p)) for p in shim_paths] +SHIM_PATHS = get_shim_paths() diff --git a/pipenv/vendor/pythonfinder/exceptions.py b/pipenv/vendor/pythonfinder/exceptions.py index df381daf..adfac16b 100644 --- a/pipenv/vendor/pythonfinder/exceptions.py +++ b/pipenv/vendor/pythonfinder/exceptions.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function class InvalidPythonVersion(Exception): diff --git a/pipenv/vendor/pythonfinder/models/mixins.py b/pipenv/vendor/pythonfinder/models/mixins.py index 7d406548..79a9d806 100644 --- a/pipenv/vendor/pythonfinder/models/mixins.py +++ b/pipenv/vendor/pythonfinder/models/mixins.py @@ -2,16 +2,60 @@ from __future__ import absolute_import, unicode_literals import abc -import attr import operator + +from collections import defaultdict + +import attr import six -from ..utils import ensure_path, KNOWN_EXTS, unnest +from cached_property import cached_property +from vistir.compat import fs_str + +from ..environment import MYPY_RUNNING +from ..exceptions import InvalidPythonVersion +from ..utils import ( + KNOWN_EXTS, Sequence, expand_paths, looks_like_python, + path_is_known_executable +) + + +if MYPY_RUNNING: + from .path import PathEntry + from .python import PythonVersion + from typing import ( + Optional, + Union, + Any, + Dict, + Iterator, + List, + DefaultDict, + Generator, + Tuple, + TypeVar, + Type, + ) + from vistir.compat import Path + + BaseFinderType = TypeVar("BaseFinderType") @attr.s class BasePath(object): + path = attr.ib(default=None) # type: Path + _children = attr.ib(default=attr.Factory(dict)) # type: Dict[str, PathEntry] + only_python = attr.ib(default=False) # type: bool + name = attr.ib(type=str) + _py_version = attr.ib(default=None) # type: Optional[PythonVersion] + _pythons = attr.ib(default=attr.Factory(defaultdict)) # type: DefaultDict[str, PathEntry] + + def __str__(self): + # type: () -> str + return fs_str("{0}".format(self.path.as_posix())) + def which(self, name): + # type: (str) -> Optional[PathEntry] """Search in this path for an executable. :param executable: The name of an executable to search for. @@ -24,26 +68,165 @@ class BasePath(object): for ext in KNOWN_EXTS ] children = self.children - found = next( - ( - children[(self.path / child).as_posix()] - for child in valid_names - if (self.path / child).as_posix() in children - ), - None, - ) + found = None + if self.path is not None: + found = next( + ( + children[(self.path / child).as_posix()] + for child in valid_names + if (self.path / child).as_posix() in children + ), + None, + ) return found + def __del__(self): + for key in ["as_python", "is_dir", "is_python", "is_executable", "py_version"]: + if key in self.__dict__: + del self.__dict__[key] + self._children = {} + for key in self._pythons.keys(): + del self._pythons[key] + + @property + def children(self): + # type: () -> Dict[str, PathEntry] + if not self.is_dir: + return {} + return self._children + + @cached_property + def as_python(self): + # type: () -> PythonVersion + py_version = None + if self.py_version: + return self.py_version + if not self.is_dir and self.is_python: + try: + from .python import PythonVersion + + py_version = PythonVersion.from_path( # type: ignore + path=self, name=self.name + ) + except (ValueError, InvalidPythonVersion): + pass + if py_version is None: + pass + return py_version # type: ignore + + @name.default + def get_name(self): + # type: () -> Optional[str] + if self.path: + return self.path.name + return None + + @cached_property + def is_dir(self): + # type: () -> bool + if not self.path: + return False + try: + ret_val = self.path.is_dir() + except OSError: + ret_val = False + return ret_val + + @cached_property + def is_executable(self): + # type: () -> bool + if not self.path: + return False + return path_is_known_executable(self.path) + + @cached_property + def is_python(self): + # type: () -> bool + if not self.path: + return False + return self.is_executable and (looks_like_python(self.path.name)) + + def get_py_version(self): + # type: () -> Optional[PythonVersion] + from ..environment import IGNORE_UNSUPPORTED + + if self.is_dir: + return None + if self.is_python: + py_version = None + from .python import PythonVersion + + try: + py_version = PythonVersion.from_path( # type: ignore + path=self, name=self.name + ) + except (InvalidPythonVersion, ValueError): + py_version = None + except Exception: + if not IGNORE_UNSUPPORTED: + raise + return py_version + return None + + @cached_property + def py_version(self): + # type: () -> Optional[PythonVersion] + if not self._py_version: + py_version = self.get_py_version() + self._py_version = py_version + else: + py_version = self._py_version + return py_version + + def _iter_pythons(self): + # type: () -> Iterator + if self.is_dir: + for entry in self.children.values(): + if entry is None: + continue + elif entry.is_dir: + for python in entry._iter_pythons(): + yield python + elif entry.is_python and entry.as_python is not None: + yield entry + elif self.is_python and self.as_python is not None: + yield self # type: ignore + + @property + def pythons(self): + # type: () -> DefaultDict[Union[str, Path], PathEntry] + if not self._pythons: + from .path import PathEntry + self._pythons = defaultdict(PathEntry) + for python in self._iter_pythons(): + python_path = python.path.as_posix() # type: ignore + self._pythons[python_path] = python + return self._pythons + + def __iter__(self): + # type: () -> Iterator + for entry in self.children.values(): + yield entry + + def __next__(self): + # type: () -> Generator + return next(iter(self)) + + def next(self): + # type: () -> Generator + return self.__next__() + def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -62,31 +245,29 @@ class BasePath(object): "find_all_python_versions" if self.is_dir else "find_python_version" ) sub_finder = operator.methodcaller( - call_method, - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + call_method, major, minor, patch, pre, dev, arch, name ) if not self.is_dir: return sub_finder(self) - path_filter = filter(None, (sub_finder(p) for p in self.children.values())) + unnested = [ + sub_finder(path) for path in expand_paths(self) + ] version_sort = operator.attrgetter("as_python.version_sort") - return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + unnested = [p for p in unnested if p is not None and p.as_python is not None] + paths = sorted(unnested, key=version_sort, reverse=True) + return list(paths) def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> Optional[PathEntry] """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -101,55 +282,63 @@ class BasePath(object): """ version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "matches", major, minor, patch, pre, dev, arch, python_name=name ) - is_py = operator.attrgetter("is_python") - py_version = operator.attrgetter("as_python") if not self.is_dir: if self.is_python and self.as_python and version_matcher(self.py_version): - return attr.evolve(self) - return - finder = ( - (child, child.as_python) - for child in unnest(self.pythons.values()) - if child.as_python - ) - py_filter = filter( - None, filter(lambda child: version_matcher(child[1]), finder) - ) - version_sort = operator.attrgetter("version_sort") - return next( - ( - c[0] - for c in sorted( - py_filter, key=lambda child: child[1].version_sort, reverse=True - ) - ), - None, + return self # type: ignore + + matching_pythons = [ + [entry, entry.as_python.version_sort] + for entry in self._iter_pythons() + if (entry is not None and entry.as_python is not None and + version_matcher(entry.py_version)) + ] + results = sorted(matching_pythons, + key=operator.itemgetter(1, 0), + reverse=True, ) + return next(iter(r[0] for r in results if r is not None), None) @six.add_metaclass(abc.ABCMeta) class BaseFinder(object): + def __init__(self): + #: Maps executable paths to PathEntries + from .path import PathEntry + + self._pythons = defaultdict(PathEntry) # type: DefaultDict[str, PathEntry] + self._versions = defaultdict(PathEntry) # type: Dict[Tuple, PathEntry] + def get_versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] """Return the available versions from the finder""" raise NotImplementedError @classmethod - def create(cls): + def create(cls, # type: Type[BaseFinderType] + *args, # type: Any + **kwargs # type: Any + ): + # type: (...) -> BaseFinderType raise NotImplementedError @property def version_paths(self): - return self.versions.values() + # type: () -> Any + return self._versions.values() @property def expanded_paths(self): + # type: () -> Any return (p.paths.values() for p in self.version_paths) + + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 221d892f..cedd8192 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -13,54 +13,87 @@ import attr import six from cached_property import cached_property - from vistir.compat import Path, fs_str -from .mixins import BasePath -from ..environment import PYENV_INSTALLED, PYENV_ROOT, ASDF_INSTALLED, ASDF_DATA_DIR +from ..environment import ( + ASDF_DATA_DIR, ASDF_INSTALLED, MYPY_RUNNING, PYENV_INSTALLED, PYENV_ROOT, + SHIM_PATHS +) from ..exceptions import InvalidPythonVersion from ..utils import ( - ensure_path, - filter_pythons, - looks_like_python, - optional_instance_of, - path_is_known_executable, - unnest, - normalize_path, - parse_pyenv_version_order, - parse_asdf_version_order + Iterable, Sequence, ensure_path, expand_paths, filter_pythons, is_in_path, + looks_like_python, normalize_path, optional_instance_of, + parse_asdf_version_order, parse_pyenv_version_order, + path_is_known_executable, unnest ) +from .mixins import BaseFinder, BasePath from .python import PythonVersion -ASDF_SHIM_PATH = normalize_path(os.path.join(ASDF_DATA_DIR, "shims")) -PYENV_SHIM_PATH = normalize_path(os.path.join(PYENV_ROOT, "shims")) -SHIM_PATHS = [ASDF_SHIM_PATH, PYENV_SHIM_PATH] +if MYPY_RUNNING: + from typing import ( + Optional, Dict, DefaultDict, Iterator, List, Union, Tuple, Generator, Callable, + Type, Any, TypeVar + ) + from .mixins import BaseFinder + from .python import PythonFinder + from .windows import WindowsFinder + FinderType = TypeVar('FinderType', BaseFinder, PythonFinder, WindowsFinder) + ChildType = Union[PythonFinder, PathEntry] + PathType = Union[PythonFinder, PathEntry] @attr.s class SystemPath(object): global_search = attr.ib(default=True) - paths = attr.ib(default=attr.Factory(defaultdict)) - _executables = attr.ib(default=attr.Factory(list)) - _python_executables = attr.ib(default=attr.Factory(list)) - path_order = attr.ib(default=attr.Factory(list)) - python_version_dict = attr.ib(default=attr.Factory(defaultdict)) - only_python = attr.ib(default=False) - pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) - asdf_finder = attr.ib(default=None) - system = attr.ib(default=False) - _version_dict = attr.ib(default=attr.Factory(defaultdict)) - ignore_unsupported = attr.ib(default=False) + paths = attr.ib(default=attr.Factory(defaultdict)) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] + _executables = attr.ib(default=attr.Factory(list)) # type: List[PathEntry] + _python_executables = attr.ib(default=attr.Factory(dict)) # type: Dict[str, PathEntry] + path_order = attr.ib(default=attr.Factory(list)) # type: List[str] + python_version_dict = attr.ib() # type: DefaultDict[Tuple, List[PythonVersion]] + only_python = attr.ib(default=False, type=bool) + pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PythonFinder")) # type: Optional[PythonFinder] + asdf_finder = attr.ib(default=None) # type: Optional[PythonFinder] + system = attr.ib(default=False, type=bool) + _version_dict = attr.ib(default=attr.Factory(defaultdict)) # type: DefaultDict[Tuple, List[PathEntry]] + ignore_unsupported = attr.ib(default=False, type=bool) - __finders = attr.ib(default=attr.Factory(dict)) + __finders = attr.ib(default=attr.Factory(dict)) # type: Dict[str, Union[WindowsFinder, PythonFinder]] def _register_finder(self, finder_name, finder): + # type: (str, Union[WindowsFinder, PythonFinder]) -> None if finder_name not in self.__finders: self.__finders[finder_name] = finder + def clear_caches(self): + for key in ["executables", "python_executables", "version_dict", "path_entries"]: + if key in self.__dict__: + del self.__dict__[key] + self._executables = [] + self._python_executables = {} + self.python_version_dict = defaultdict(list) + self._version_dict = defaultdict(list) + + def __del__(self): + self.clear_caches() + self.path_order = [] + self.pyenv_finder = None + self.asdf_finder = None + self.paths = defaultdict(PathEntry) + + @property + def finders(self): + # type: () -> List[str] + return [k for k in self.__finders.keys()] + + @python_version_dict.default + def create_python_version_dict(self): + # type: () -> DefaultDict[Tuple, List[PythonVersion]] + return defaultdict(list) + @cached_property def executables(self): + # type: () -> List[PathEntry] self.executables = [ p for p in chain(*(child.children.values() for child in self.paths.values())) @@ -70,6 +103,7 @@ class SystemPath(object): @cached_property def python_executables(self): + # type: () -> Dict[str, PathEntry] python_executables = {} for child in self.paths.values(): if child.pythons: @@ -82,31 +116,28 @@ class SystemPath(object): @cached_property def version_dict(self): - self._version_dict = defaultdict(list) + # type: () -> DefaultDict[Tuple, List[PathEntry]] + self._version_dict = defaultdict(list) # type: DefaultDict[Tuple, List[PathEntry]] for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): if finder_name == "windows": if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue - if type(entry).__name__ == "VersionPath": - for path in entry.paths.values(): - if path not in self._version_dict[version] and path.is_python: - self._version_dict[version].append(path) - continue - continue - elif entry not in self._version_dict[version] and entry.is_python: + if entry not in self._version_dict[version] and entry.is_python: self._version_dict[version].append(entry) for p, entry in self.python_executables.items(): version = entry.as_python if not version: continue - version = version.version_tuple + if not isinstance(version, tuple): + version = version.version_tuple if version and entry not in self._version_dict[version]: self._version_dict[version].append(entry) return self._version_dict def __attrs_post_init__(self): + # type: () -> None #: slice in pyenv if not self.__class__ == SystemPath: return @@ -124,7 +155,7 @@ class SystemPath(object): if venv and (self.system or self.global_search): p = ensure_path(venv) self.path_order = [(p / bin_dir).as_posix()] + self.path_order - self.paths[p] = PathEntry.create(path=p, is_root=True, only_python=False) + self.paths[p] = self.get_path(p.joinpath(bin_dir)) if self.system: syspath = Path(sys.executable) syspath_bin = syspath.parent @@ -136,26 +167,35 @@ class SystemPath(object): ) def _get_last_instance(self, path): + # type: (str) -> int reversed_paths = reversed(self.path_order) paths = [normalize_path(p) for p in reversed_paths] normalized_target = normalize_path(path) last_instance = next( iter(p for p in paths if normalized_target in p), None ) - try: - path_index = self.path_order.index(last_instance) - except ValueError: - return + if last_instance is None: + raise ValueError("No instance found on path for target: {0!s}".format(path)) + path_index = self.path_order.index(last_instance) return path_index def _slice_in_paths(self, start_idx, paths): - before_path = self.path_order[: start_idx + 1] - after_path = self.path_order[start_idx + 2 :] + # type: (int, List[Path]) -> None + before_path = [] # type: List[str] + after_path = [] # type: List[str] + if start_idx == 0: + after_path = self.path_order[:] + elif start_idx == -1: + before_path = self.path_order[:] + else: + before_path = self.path_order[: start_idx + 1] + after_path = self.path_order[start_idx + 2 :] self.path_order = ( before_path + [p.as_posix() for p in paths] + after_path ) def _remove_path(self, path): + # type: (str) -> None path_copy = [p for p in reversed(self.path_order[:])] new_order = [] target = normalize_path(path) @@ -164,7 +204,7 @@ class SystemPath(object): for pth in self.paths.keys() } if target in path_map: - del self.paths[path_map.get(target)] + del self.paths[path_map[target]] for current_path in path_copy: normalized = normalize_path(current_path) if normalized != target: @@ -173,41 +213,80 @@ class SystemPath(object): self.path_order = new_order def _setup_asdf(self): + # type: () -> None from .python import PythonFinder + os_path = os.environ["PATH"].split(os.pathsep) self.asdf_finder = PythonFinder.create( root=ASDF_DATA_DIR, ignore_unsupported=True, sort_function=parse_asdf_version_order, version_glob_path="installs/python/*") - asdf_index = self._get_last_instance(ASDF_DATA_DIR) - if not asdf_index: + asdf_index = None + try: + asdf_index = self._get_last_instance(ASDF_DATA_DIR) + except ValueError: + pyenv_index = 0 if is_in_path(next(iter(os_path), ""), PYENV_ROOT) else -1 + if asdf_index is None: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here return root_paths = [p for p in self.asdf_finder.roots] - self._slice_in_paths(asdf_index, root_paths) + self._slice_in_paths(asdf_index, [self.asdf_finder.root]) + self.paths[self.asdf_finder.root] = self.asdf_finder self.paths.update(self.asdf_finder.roots) self._remove_path(normalize_path(os.path.join(ASDF_DATA_DIR, "shims"))) self._register_finder("asdf", self.asdf_finder) + def reload_finder(self, finder_name): + # type: (str) -> None + if finder_name is None: + raise TypeError("Must pass a string as the name of the target finder") + finder_attr = "{0}_finder".format(finder_name) + setup_attr = "_setup_{0}".format(finder_name) + try: + current_finder = getattr(self, finder_attr) # type: Any + except AttributeError: + raise ValueError("Must pass a valid finder to reload.") + try: + setup_fn = getattr(self, setup_attr) + except AttributeError: + raise ValueError("Finder has no valid setup function: %s" % finder_name) + if current_finder is None: + # TODO: This is called 'reload', should we load a new finder for the first + # time here? lets just skip that for now to avoid unallowed finders + pass + if (finder_name == "pyenv" and not PYENV_INSTALLED) or (finder_name == "asdf" and not ASDF_INSTALLED): + # Don't allow loading of finders that aren't explicitly 'installed' as it were + pass + setattr(self, finder_attr, None) + if finder_name in self.__finders: + del self.__finders[finder_name] + setup_fn() + def _setup_pyenv(self): + # type: () -> None from .python import PythonFinder + os_path = os.environ["PATH"].split(os.pathsep) self.pyenv_finder = PythonFinder.create( - root=PYENV_ROOT, sort_function=parse_pyenv_version_order, - version_glob_path="versions/*", ignore_unsupported=self.ignore_unsupported - ) - pyenv_index = self._get_last_instance(PYENV_ROOT) - if not pyenv_index: + root=PYENV_ROOT, sort_function=parse_pyenv_version_order, version_glob_path="versions/*", ignore_unsupported=self.ignore_unsupported) + pyenv_index = None + try: + pyenv_index = self._get_last_instance(PYENV_ROOT) + except ValueError: + pyenv_index = 0 if is_in_path(next(iter(os_path), ""), PYENV_ROOT) else -1 + if pyenv_index is None: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here return - root_paths = [p for p in self.pyenv_finder.roots] - self._slice_in_paths(pyenv_index, root_paths) + root_paths = [p for p in self.pyenv_finder.roots] + self._slice_in_paths(pyenv_index, [self.pyenv_finder.root]) + self.paths[self.pyenv_finder.root] = self.pyenv_finder self.paths.update(self.pyenv_finder.roots) self._remove_path(os.path.join(PYENV_ROOT, "shims")) self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): + # type: () -> None from .windows import WindowsFinder self.windows_finder = WindowsFinder.create() @@ -218,6 +297,9 @@ class SystemPath(object): self._register_finder("windows", self.windows_finder) def get_path(self, path): + # type: (Union[str, Path]) -> PathType + if path is None: + raise TypeError("A path must be provided in order to generate a path entry.") path = ensure_path(path) _path = self.paths.get(path) if not _path: @@ -227,69 +309,90 @@ class SystemPath(object): path=path.absolute(), is_root=True, only_python=self.only_python ) self.paths[path.as_posix()] = _path + if not _path: + raise ValueError("Path not found or generated: {0!r}".format(path)) return _path def _get_paths(self): - return (self.get_path(k) for k in self.path_order) + # type: () -> Iterator + for path in self.path_order: + try: + entry = self.get_path(path) + except ValueError: + continue + else: + yield entry @cached_property def path_entries(self): - paths = self._get_paths() + # type: () -> List[Union[PathEntry, FinderType]] + paths = list(self._get_paths()) return paths def find_all(self, executable): - """Search the path for an executable. Return all copies. + # type: (str) -> List[Union[PathEntry, FinderType]] + """ + Search the path for an executable. Return all copies. :param executable: Name of the executable :type executable: str :returns: List[PathEntry] """ - sub_which = operator.methodcaller("which", name=executable) + + sub_which = operator.methodcaller("which", executable) filtered = (sub_which(self.get_path(k)) for k in self.path_order) return list(filtered) def which(self, executable): - """Search for an executable on the path. + # type: (str) -> Union[PathEntry, None] + """ + Search for an executable on the path. :param executable: Name of the executable to be located. :type executable: str :returns: :class:`~pythonfinder.models.PathEntry` object. """ - sub_which = operator.methodcaller("which", name=executable) + + sub_which = operator.methodcaller("which", executable) filtered = (sub_which(self.get_path(k)) for k in self.path_order) return next(iter(f for f in filtered if f is not None), None) def _filter_paths(self, finder): - return ( - pth for pth in unnest(finder(p) for p in self.path_entries if p is not None) - if pth is not None - ) + # type: (Callable) -> Iterator + for path in self._get_paths(): + if path is None: + continue + python_versions = finder(path) + if python_versions is not None: + for python in python_versions: + if python is not None: + yield python def _get_all_pythons(self, finder): - paths = {p.path.as_posix(): p for p in self._filter_paths(finder)} - paths.update(self.python_executables) - return (p for p in paths.values() if p is not None) + # type: (Callable) -> Iterator + for python in self._filter_paths(finder): + if python is not None and python.is_python: + yield python def get_pythons(self, finder): + # type: (Callable) -> Iterator sort_key = operator.attrgetter("as_python.version_sort") - return ( - k for k in sorted( - (p for p in self._filter_paths(finder) if p.is_python), - key=sort_key, - reverse=True - ) if k is not None - ) + pythons = [entry for entry in self._get_all_pythons(finder)] + for python in sorted(pythons, key=sort_key, reverse=True): + if python is not None: + yield python def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -305,21 +408,12 @@ class SystemPath(object): """ sub_finder = operator.methodcaller( - "find_all_python_versions", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "find_all_python_versions", major, minor, patch, pre, dev, arch, name ) alternate_sub_finder = None if major and not (minor or patch or pre or dev or arch or name): alternate_sub_finder = operator.methodcaller( - "find_all_python_versions", - major=None, - name=major + "find_all_python_versions", None, None, None, None, None, None, major ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) @@ -332,14 +426,15 @@ class SystemPath(object): def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[Union[str, int]] + patch=None, # type: Optional[Union[str, int]] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> PathEntry """Search for a specific python version on the path. :param major: Major python version to search for. @@ -356,33 +451,31 @@ class SystemPath(object): if isinstance(major, six.string_types) and not minor and not patch: # Only proceed if this is in the format "x.y.z" or similar - if major.count(".") > 0 and major[0].isdigit(): + if major.isdigit() or (major.count(".") > 0 and major[0].isdigit()): version = major.split(".", 2) - if len(version) > 3: - major, minor, patch, rest = version - elif len(version) == 3: - major, minor, patch = version + if isinstance(version, (tuple, list)): + if len(version) > 3: + major, minor, patch, rest = version + elif len(version) == 3: + major, minor, patch = version + elif len(version) == 2: + major, minor = version + else: + major = major[0] else: - major, minor = version + major = major + name = None else: name = "{0!s}".format(major) major = None sub_finder = operator.methodcaller( "find_python_version", - major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + major, minor, patch, pre, dev, arch, name, ) alternate_sub_finder = None - if major and not (minor or patch or pre or dev or arch or name): + if name and not (minor or patch or pre or dev or arch or major): alternate_sub_finder = operator.methodcaller( - "find_all_python_versions", - major=None, - name=major + "find_all_python_versions", None, None, None, None, None, None, name ) if major and minor and patch: _tuple_pre = pre if pre is not None else False @@ -406,12 +499,13 @@ class SystemPath(object): @classmethod def create( cls, - path=None, - system=False, - only_python=False, - global_search=True, - ignore_unsupported=True, + path=None, # type: str + system=False, # type: bool + only_python=False, # type: bool + global_search=True, # type: bool + ignore_unsupported=True, # type: bool ): + # type: (...) -> SystemPath """Create a new :class:`pythonfinder.models.SystemPath` instance. :param path: Search path to prepend when searching, defaults to None @@ -423,14 +517,16 @@ class SystemPath(object): :rtype: :class:`pythonfinder.models.SystemPath` """ - path_entries = defaultdict(PathEntry) - paths = [] + path_entries = defaultdict(PathEntry) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] + paths = [] # type: List[str] if ignore_unsupported: os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1") if global_search: - paths = os.environ.get("PATH").split(os.pathsep) + if "PATH" in os.environ: + paths = os.environ["PATH"].split(os.pathsep) if path: paths = [path] + paths + paths = [p for p in paths if not any(is_in_path(p, shim) for shim in SHIM_PATHS)] _path_objects = [ensure_path(p.strip('"')) for p in paths] paths = [p.as_posix() for p in _path_objects] path_entries.update( @@ -439,7 +535,6 @@ class SystemPath(object): path=p.absolute(), is_root=True, only_python=only_python ) for p in _path_objects - if not any(shim in normalize_path(str(p)) for shim in SHIM_PATHS) } ) return cls( @@ -454,18 +549,15 @@ class SystemPath(object): @attr.s(slots=True) class PathEntry(BasePath): - path = attr.ib(default=None, validator=optional_instance_of(Path)) - _children = attr.ib(default=attr.Factory(dict)) - is_root = attr.ib(default=True) - only_python = attr.ib(default=False) - name = attr.ib() - py_version = attr.ib() - _pythons = attr.ib(default=attr.Factory(defaultdict)) + is_root = attr.ib(default=True, type=bool) - def __str__(self): - return fs_str("{0}".format(self.path.as_posix())) + def __del__(self): + if "children" in self.__dict__: + del self.__dict__["children"] + BasePath.__del__(self) def _filter_children(self): + # type: () -> Iterator[Path] if self.only_python: children = filter_pythons(self.path) else: @@ -473,86 +565,47 @@ class PathEntry(BasePath): return children def _gen_children(self): + # type: () -> Iterator + from ..environment import get_shim_paths + shim_paths = get_shim_paths() pass_name = self.name != self.path.name pass_args = {"is_root": False, "only_python": self.only_python} if pass_name: - pass_args["name"] = self.name + if self.name is not None and isinstance(self.name, six.string_types): + pass_args["name"] = self.name # type: ignore + elif self.path is not None and isinstance(self.path.name, six.string_types): + pass_args["name"] = self.path.name # type: ignore if not self.is_dir: - yield (self.path.as_posix(), copy.deepcopy(self)) + yield (self.path.as_posix(), self) elif self.is_root: for child in self._filter_children(): - if any(shim in normalize_path(str(child)) for shim in SHIM_PATHS): + if any(is_in_path(str(child), shim) for shim in shim_paths): continue if self.only_python: try: - entry = PathEntry.create(path=child, **pass_args) + entry = PathEntry.create(path=child, **pass_args) # type: ignore except (InvalidPythonVersion, ValueError): continue else: - entry = PathEntry.create(path=child, **pass_args) + entry = PathEntry.create(path=child, **pass_args) # type: ignore yield (child.as_posix(), entry) return @cached_property def children(self): - if not self._children: - children = {} + # type: () -> Dict[str, PathEntry] + children = getattr(self, "_children", {}) # type: Dict[str, PathEntry] + if not children: for child_key, child_val in self._gen_children(): children[child_key] = child_val self._children = children return self._children - @name.default - def get_name(self): - return self.path.name - - @py_version.default - def get_py_version(self): - from ..environment import IGNORE_UNSUPPORTED - if self.is_dir: - return None - if self.is_python: - py_version = None - try: - py_version = PythonVersion.from_path(path=self, name=self.name) - except (InvalidPythonVersion, ValueError): - py_version = None - except Exception: - if not IGNORE_UNSUPPORTED: - raise - return py_version - return - - @property - def pythons(self): - if not self._pythons: - if self.is_dir: - for path, entry in self.children.items(): - _path = ensure_path(entry.path) - if entry.is_python: - self._pythons[_path.as_posix()] = entry - else: - if self.is_python: - _path = ensure_path(self.path) - self._pythons[_path.as_posix()] = self - return self._pythons - - @cached_property - def as_python(self): - py_version = None - if self.py_version: - return self.py_version - if not self.is_dir and self.is_python: - try: - from .python import PythonVersion - py_version = PythonVersion.from_path(path=attr.evolve(self), name=self.name) - except (ValueError, InvalidPythonVersion): - py_version = None - return py_version @classmethod def create(cls, path, is_root=False, only_python=False, pythons=None, name=None): + # type: (Union[str, Path], bool, bool, Dict[str, PythonVersion], Optional[str]) -> PathEntry """Helper method for creating new :class:`pythonfinder.models.PathEntry` instances. :param str path: Path to the specified location. @@ -580,12 +633,12 @@ class PathEntry(BasePath): "only_python": only_python } if not guessed_name: - child_creation_args["name"] = name + child_creation_args["name"] = _new.name # type: ignore for pth, python in pythons.items(): if any(shim in normalize_path(str(pth)) for shim in SHIM_PATHS): continue pth = ensure_path(pth) - children[pth.as_posix()] = PathEntry( + children[pth.as_posix()] = PathEntry( # type: ignore py_version=python, path=pth, **child_creation_args @@ -593,29 +646,11 @@ class PathEntry(BasePath): _new._children = children return _new - @cached_property - def is_dir(self): - try: - ret_val = self.path.is_dir() - except OSError: - ret_val = False - return ret_val - - @cached_property - def is_executable(self): - return path_is_known_executable(self.path) - - @cached_property - def is_python(self): - return self.is_executable and ( - looks_like_python(self.path.name) - ) - @attr.s class VersionPath(SystemPath): - base = attr.ib(default=None, validator=optional_instance_of(Path)) - name = attr.ib(default=None) + base = attr.ib(default=None, validator=optional_instance_of(Path)) # type: Path + name = attr.ib(default=None) # type: str @classmethod def create(cls, path, only_python=True, pythons=None, name=None): diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 4fcbbca6..987a830e 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -2,53 +2,71 @@ from __future__ import absolute_import, print_function import copy -import platform -import operator import logging +import operator +import platform +import sys from collections import defaultdict import attr +import six -from packaging.version import Version, LegacyVersion -from packaging.version import parse as parse_version -from vistir.compat import Path +from packaging.version import Version +from vistir.compat import Path, lru_cache -from ..environment import SYSTEM_ARCH, PYENV_ROOT, ASDF_DATA_DIR +from ..environment import ASDF_DATA_DIR, MYPY_RUNNING, PYENV_ROOT, SYSTEM_ARCH from ..exceptions import InvalidPythonVersion -from .mixins import BaseFinder, BasePath from ..utils import ( - _filter_none, - ensure_path, - get_python_version, - optional_instance_of, - unnest, - is_in_path, - parse_pyenv_version_order, - parse_asdf_version_order, - parse_python_version, + RE_MATCHER, _filter_none, ensure_path, get_python_version, is_in_path, + looks_like_python, optional_instance_of, parse_asdf_version_order, + parse_pyenv_version_order, parse_python_version, unnest ) +from .mixins import BaseFinder, BasePath + + +if MYPY_RUNNING: + from typing import ( + DefaultDict, Optional, Callable, Generator, Any, Union, Tuple, List, Dict, Type, + TypeVar, Iterator + ) + from .path import PathEntry + from .._vendor.pep514tools.environment import Environment + logger = logging.getLogger(__name__) @attr.s(slots=True) class PythonFinder(BaseFinder, BasePath): - root = attr.ib(default=None, validator=optional_instance_of(Path)) - #: ignore_unsupported should come before versions, because its value is used - #: in versions's default initializer. - ignore_unsupported = attr.ib(default=True) - #: The function to use to sort version order when returning an ordered verion set - sort_function = attr.ib(default=None) - paths = attr.ib(default=attr.Factory(list)) - roots = attr.ib(default=attr.Factory(defaultdict)) + root = attr.ib(default=None, validator=optional_instance_of(Path), type=Path) + # should come before versions, because its value is used in versions's default initializer. + #: Whether to ignore any paths which raise exceptions and are not actually python + ignore_unsupported = attr.ib(default=True, type=bool) #: Glob path for python versions off of the root directory - version_glob_path = attr.ib(default="versions/*") - versions = attr.ib() - pythons = attr.ib() + version_glob_path = attr.ib(default="versions/*", type=str) + #: The function to use to sort version order when returning an ordered verion set + sort_function = attr.ib(default=None) # type: Callable + #: The root locations used for discovery + roots = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + #: List of paths discovered during search + paths = attr.ib(type=list) + #: shim directory + shim_dir = attr.ib(default="shims", type=str) + #: Versions discovered in the specified paths + _versions = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + _pythons = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + + def __del__(self): + # type: () -> None + self._versions = defaultdict() + self._pythons = defaultdict() + self.roots = defaultdict() + self.paths = [] @property def expanded_paths(self): + # type: () -> Generator return ( path for path in unnest(p for p in self.versions.values()) if path is not None @@ -56,18 +74,22 @@ class PythonFinder(BaseFinder, BasePath): @property def is_pyenv(self): + # type: () -> bool return is_in_path(str(self.root), PYENV_ROOT) @property def is_asdf(self): + # type: () -> bool return is_in_path(str(self.root), ASDF_DATA_DIR) def get_version_order(self): + # type: () -> List[Path] version_paths = [ p for p in self.root.glob(self.version_glob_path) if not (p.parent.name == "envs" or p.name == "envs") ] versions = {v.name: v for v in version_paths} + version_order = [] # type: List[Path] if self.is_pyenv: version_order = [versions[v] for v in parse_pyenv_version_order() if v in versions] elif self.is_asdf: @@ -80,91 +102,129 @@ class PythonFinder(BaseFinder, BasePath): version_order = version_paths return version_order + def get_bin_dir(self, base): + # type: (Union[Path, str]) -> Path + if isinstance(base, six.string_types): + base = Path(base) + return base / "bin" + @classmethod - def version_from_bin_dir(cls, base_dir, name=None): - from .path import PathEntry + def version_from_bin_dir(cls, entry): + # type: (PathEntry) -> Optional[PathEntry] py_version = None - version_path = PathEntry.create( - path=base_dir.absolute().as_posix(), - only_python=True, - name=base_dir.parent.name, - ) - py_version = next(iter(version_path.find_all_python_versions()), None) + py_version = next(iter(entry.find_all_python_versions()), None) return py_version - @versions.default - def get_versions(self): + def _iter_version_bases(self): + # type: () -> Iterator[Tuple[Path, PathEntry]] from .path import PathEntry - versions = defaultdict() - bin_ = "{base}/bin" for p in self.get_version_order(): - bin_dir = Path(bin_.format(base=p.as_posix())) - version_path = None - if bin_dir.exists(): - version_path = PathEntry.create( - path=bin_dir.absolute().as_posix(), - only_python=False, - name=p.name, - is_root=True, + bin_dir = self.get_bin_dir(p) + if bin_dir.exists() and bin_dir.is_dir(): + entry = PathEntry.create( + path=bin_dir.absolute(), only_python=False, name=p.name, + is_root=True ) + self.roots[p] = entry + yield (p, entry) + + def _iter_versions(self): + # type: () -> Iterator[Tuple[Path, PathEntry, Tuple]] + for base_path, entry in self._iter_version_bases(): version = None + version_entry = None try: - version = PythonVersion.parse(p.name) + version = PythonVersion.parse(entry.name) except (ValueError, InvalidPythonVersion): - entry = next(iter(version_path.find_all_python_versions()), None) - if not entry: - if self.ignore_unsupported: - continue - raise - else: - version = entry.py_version.as_dict() + version_entry = next(iter(entry.find_all_python_versions()), None) + if version is None: + if not self.ignore_unsupported: + raise + continue + if version_entry is not None: + version = version_entry.py_version.as_dict() except Exception: if not self.ignore_unsupported: raise - logger.warning( - "Unsupported Python version %r, ignoring...", p.name, exc_info=True + logger.warning("Unsupported Python version %r, ignoring...", + base_path.name, exc_info=True) + continue + if version is not None: + version_tuple = ( + version.get("major"), + version.get("minor"), + version.get("patch"), + version.get("is_prerelease"), + version.get("is_devrelease"), + version.get("is_debug"), ) - continue - if not version: - continue - version_tuple = ( - version.get("major"), - version.get("minor"), - version.get("patch"), - version.get("is_prerelease"), - version.get("is_devrelease"), - version.get("is_debug"), - ) - self.roots[p] = version_path - versions[version_tuple] = version_path - self.paths.append(version_path) - return versions + yield (base_path, entry, version_tuple) + + @property + def versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] + if not self._versions: + for base_path, entry, version_tuple in self._iter_versions(): + self._versions[version_tuple] = entry + return self._versions + + def _iter_pythons(self): + # type: () -> Iterator + for path, entry, version_tuple in self._iter_versions(): + if path.as_posix() in self._pythons: + yield self._pythons[path.as_posix()] + elif version_tuple not in self.versions: + for python in entry.find_all_python_versions(): + yield python + else: + yield self.versions[version_tuple] + + @paths.default + def get_paths(self): + # type: () -> List[PathEntry] + _paths = [base for _, base in self._iter_version_bases()] + return _paths + + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + if not self._pythons: + from .path import PathEntry + self._pythons = defaultdict(PathEntry) # type: DefaultDict[str, PathEntry] + for python in self._iter_pythons(): + python_path = python.path.as_posix() # type: ignore + self._pythons[python_path] = python + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value - @pythons.default def get_pythons(self): - pythons = defaultdict() - for p in self.paths: - pythons.update(p.pythons) - return pythons + # type: () -> DefaultDict[str, PathEntry] + return self.pythons @classmethod - def create(cls, root, sort_function=None, version_glob_path=None, ignore_unsupported=True): + def create(cls, root, sort_function, version_glob_path=None, ignore_unsupported=True): # type: ignore + # type: (Type[PythonFinder], str, Callable, Optional[str], bool) -> PythonFinder root = ensure_path(root) if not version_glob_path: version_glob_path = "versions/*" - return cls(root=root, ignore_unsupported=ignore_unsupported, + return cls(root=root, path=root, ignore_unsupported=ignore_unsupported, # type: ignore sort_function=sort_function, version_glob_path=version_glob_path) def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -179,36 +239,40 @@ class PythonFinder(BaseFinder, BasePath): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + call_method = ( + "find_all_python_versions" if self.is_dir else "find_python_version" ) - py = operator.attrgetter("as_python") - pythons = ( - py_ver for py_ver in (py(p) for p in self.pythons.values() if p is not None) - if py_ver is not None + sub_finder = operator.methodcaller( + call_method, major, minor, patch, pre, dev, arch, name ) - # pythons = filter(None, [p.as_python for p in self.pythons.values()]) - matching_versions = filter(lambda py: version_matcher(py), pythons) - version_sort = operator.attrgetter("version_sort") - return sorted(matching_versions, key=version_sort, reverse=True) + if not any([major, minor, patch, name]): + pythons = [ + next(iter(py for py in base.find_all_python_versions()), None) + for _, base in self._iter_version_bases() + ] + else: + pythons = [ + sub_finder(path) for path in self.paths + ] + pythons = [p for p in pythons if p and p.is_python and p.as_python is not None] + version_sort = operator.attrgetter("as_python.version_sort") + paths = [ + p for p in sorted(list(pythons), key=version_sort, reverse=True) + if p is not None + ] + return paths def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> Optional[PathEntry] """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -222,40 +286,66 @@ class PythonFinder(BaseFinder, BasePath): :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ - version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + sub_finder = operator.methodcaller( + "find_python_version", major, minor, patch, pre, dev, arch, name ) - pythons = filter(None, [p.as_python for p in self.pythons.values()]) - matching_versions = filter(lambda py: version_matcher(py), pythons) - version_sort = operator.attrgetter("version_sort") - return next(iter(c for c in sorted(matching_versions, key=version_sort, reverse=True)), None) + version_sort = operator.attrgetter("as_python.version_sort") + unnested = [sub_finder(self.roots[path]) for path in self.roots] + unnested = [ + p for p in unnested + if p is not None and p.is_python and p.as_python is not None + ] + paths = sorted(list(unnested), key=version_sort, reverse=True) + return next(iter(p for p in paths if p is not None), None) + + def which(self, name): + # type: (str) -> Optional[PathEntry] + """Search in this path for an executable. + + :param executable: The name of an executable to search for. + :type executable: str + :returns: :class:`~pythonfinder.models.PathEntry` instance. + """ + + matches = (p.which(name) for p in self.paths) + non_empty_match = next(iter(m for m in matches if m is not None), None) + return non_empty_match @attr.s(slots=True) class PythonVersion(object): - major = attr.ib(default=0) - minor = attr.ib(default=None) - patch = attr.ib(default=0) - is_prerelease = attr.ib(default=False) - is_postrelease = attr.ib(default=False) - is_devrelease = attr.ib(default=False) - is_debug = attr.ib(default=False) - version = attr.ib(default=None) - architecture = attr.ib(default=None) - comes_from = attr.ib(default=None) - executable = attr.ib(default=None) - name = attr.ib(default=None) + major = attr.ib(default=0, type=int) + minor = attr.ib(default=None) # type: Optional[int] + patch = attr.ib(default=0) # type: Optional[int] + is_prerelease = attr.ib(default=False, type=bool) + is_postrelease = attr.ib(default=False, type=bool) + is_devrelease = attr.ib(default=False, type=bool) + is_debug = attr.ib(default=False, type=bool) + version = attr.ib(default=None) # type: Version + architecture = attr.ib(default=None) # type: Optional[str] + comes_from = attr.ib(default=None) # type: Optional[PathEntry] + executable = attr.ib(default=None) # type: Optional[str] + name = attr.ib(default=None, type=str) + + def __getattribute__(self, key): + result = super(PythonVersion, self).__getattribute__(key) + if key in ["minor", "patch"] and result is None: + executable = None # type: Optional[str] + if self.executable: + executable = self.executable + elif self.comes_from: + executable = self.comes_from.path.as_posix() + if executable is not None: + instance_dict = self.parse_executable(executable) + self.update_metadata(instance_dict) + result = instance_dict.get(key) + return result @property def version_sort(self): - """version_sort tuple for sorting against other instances of the same class. + # type: () -> Tuple[Optional[int], Optional[int], int, int] + """ + A tuple for sorting against other instances of the same class. Returns a tuple of the python version but includes a point for non-dev, and a point for non-prerelease versions. So released versions will have 2 points @@ -275,7 +365,9 @@ class PythonVersion(object): @property def version_tuple(self): - """Provides a version tuple for using as a dictionary key. + # type: () -> Tuple[int, Optional[int], Optional[int], bool, bool, bool] + """ + Provides a version tuple for using as a dictionary key. :return: A tuple describing the python version meetadata contained. :rtype: tuple @@ -292,45 +384,52 @@ class PythonVersion(object): def matches( self, - major=None, - minor=None, - patch=None, - pre=False, - dev=False, - arch=None, - debug=False, - name=None, + major=None, # type: Optional[int] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=False, # type: bool + dev=False, # type: bool + arch=None, # type: Optional[str] + debug=False, # type: bool + python_name=None, # type: Optional[str] ): + # type: (...) -> bool + result = False if arch: own_arch = self.get_architecture() if arch.isdigit(): arch = "{0}bit".format(arch) - return ( - (major is None or self.major == major) - and (minor is None or self.minor == minor) - and (patch is None or self.patch == patch) + if ( + (major is None or self.major and self.major == major) + and (minor is None or self.minor and self.minor == minor) + and (patch is None or self.patch and self.patch == patch) and (pre is None or self.is_prerelease == pre) and (dev is None or self.is_devrelease == dev) and (arch is None or own_arch == arch) and (debug is None or self.is_debug == debug) and ( - name is None - or (name and self.name) - and (self.name == name or self.name.startswith(name)) + python_name is None + or (python_name and self.name) + and (self.name == python_name or self.name.startswith(python_name)) ) - ) + ): + result = True + return result def as_major(self): + # type: () -> PythonVersion self_dict = attr.asdict(self, recurse=False, filter=_filter_none).copy() self_dict.update({"minor": None, "patch": None}) return self.create(**self_dict) def as_minor(self): + # type: () -> PythonVersion self_dict = attr.asdict(self, recurse=False, filter=_filter_none).copy() self_dict.update({"patch": None}) return self.create(**self_dict) def as_dict(self): + # type: () -> Dict[str, Union[int, bool, Version, None]] return { "major": self.major, "minor": self.minor, @@ -342,35 +441,67 @@ class PythonVersion(object): "version": self.version, } + def update_metadata(self, metadata): + # type: (Dict[str, Union[str, int, Version]]) -> None + """ + Update the metadata on the current :class:`pythonfinder.models.python.PythonVersion` + + Given a parsed version dictionary from :func:`pythonfinder.utils.parse_python_version`, + update the instance variables of the current version instance to reflect the newly + supplied values. + """ + + for key in metadata: + try: + current_value = getattr(self, key) + except AttributeError: + continue + else: + setattr(self, key, metadata[key]) + @classmethod + @lru_cache(maxsize=1024) def parse(cls, version): - """Parse a valid version string into a dictionary + # type: (str) -> Dict[str, Union[str, int, Version]] + """ + Parse a valid version string into a dictionary Raises: ValueError -- Unable to parse version string ValueError -- Not a valid python version + TypeError -- NoneType or unparseable type passed in - :param version: A valid version string - :type version: str + :param str version: A valid version string :return: A dictionary with metadata about the specified python version. - :rtype: dict. + :rtype: dict """ + if version is None: + raise TypeError("Must pass a value to parse!") version_dict = parse_python_version(str(version)) if not version_dict: raise ValueError("Not a valid python version: %r" % version) return version_dict def get_architecture(self): + # type: () -> str if self.architecture: return self.architecture - arch, _ = platform.architecture(self.comes_from.path.as_posix()) + arch = None + if self.comes_from is not None: + arch, _ = platform.architecture(self.comes_from.path.as_posix()) + elif self.executable is not None: + arch, _ = platform.architecture(self.executable) + if arch is None: + arch, _ = platform.architecture(sys.executable) self.architecture = arch return self.architecture @classmethod def from_path(cls, path, name=None, ignore_unsupported=True): - """Parses a python version from a system path. + # type: (Union[str, PathEntry], Optional[str], bool) -> PythonVersion + """ + Parses a python version from a system path. Raises: ValueError -- Not a valid python path @@ -389,22 +520,49 @@ class PythonVersion(object): path = PathEntry.create(path, is_root=False, only_python=True, name=name) from ..environment import IGNORE_UNSUPPORTED ignore_unsupported = ignore_unsupported or IGNORE_UNSUPPORTED + path_name = getattr(path, "name", path.path.name) # str if not path.is_python: if not (ignore_unsupported or IGNORE_UNSUPPORTED): raise ValueError("Not a valid python path: %s" % path.path) - py_version = get_python_version(path.path.absolute().as_posix()) - instance_dict = cls.parse(py_version.strip()) + try: + instance_dict = cls.parse(path_name) + except Exception: + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + else: + if instance_dict.get("minor") is None and looks_like_python(path.path.name): + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + if not isinstance(instance_dict.get("version"), Version) and not ignore_unsupported: - raise ValueError("Not a valid python path: %s" % path.path) - if not name: - name = path.name + raise ValueError("Not a valid python path: %s" % path) + if instance_dict.get("patch") is None: + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + if name is None: + name = path_name instance_dict.update( - {"comes_from": path, "name": name} + {"comes_from": path, "name": name, "executable": path.path.as_posix()} ) - return cls(**instance_dict) + return cls(**instance_dict) # type: ignore + + @classmethod + @lru_cache(maxsize=1024) + def parse_executable(cls, path): + # type: (str) -> Dict[str, Optional[Union[str, int, Version]]] + result_dict = {} # type: Dict[str, Optional[Union[str, int, Version]]] + result_version = None # type: Optional[str] + if path is None: + raise TypeError("Must pass a valid path to parse.") + try: + result_version = get_python_version(path) + except Exception: + raise ValueError("Not a valid python path: %r" % path) + if result_version is None: + raise ValueError("Not a valid python path: %s" % path) + result_dict = cls.parse(result_version.strip()) + return result_dict @classmethod def from_windows_launcher(cls, launcher_entry, name=None): + # type: (Environment, Optional[str]) -> PythonVersion """Create a new PythonVersion instance from a Windows Launcher Entry :param launcher_entry: A python launcher environment object. @@ -440,6 +598,7 @@ class PythonVersion(object): @classmethod def create(cls, **kwargs): + # type: (...) -> PythonVersion if "architecture" in kwargs: if kwargs["architecture"].isdigit(): kwargs["architecture"] = "{0}bit".format(kwargs["architecture"]) @@ -448,10 +607,11 @@ class PythonVersion(object): @attr.s class VersionMap(object): - versions = attr.ib(default=attr.Factory(defaultdict(list))) + versions = attr.ib(factory=defaultdict) # type: DefaultDict[Tuple[int, Optional[int], Optional[int], bool, bool, bool], List[PathEntry]] def add_entry(self, entry): - version = entry.as_python + # type: (...) -> None + version = entry.as_python # type: PythonVersion if version: entries = self.versions[version.version_tuple] paths = {p.path for p in self.versions.get(version.version_tuple, [])} @@ -459,13 +619,18 @@ class VersionMap(object): self.versions[version.version_tuple].append(entry) def merge(self, target): + # type: (VersionMap) -> None for version, entries in target.versions.items(): if version not in self.versions: self.versions[version] = entries else: - current_entries = {p.path for p in self.versions.get(version)} + current_entries = { + p.path for p in + self.versions[version] # type: ignore + if version in self.versions + } new_entries = {p.path for p in entries} new_entries -= current_entries - self.versions[version].append( + self.versions[version].extend( [e for e in entries if e.path in new_entries] ) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index f985630f..1478c600 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -7,6 +7,7 @@ from collections import defaultdict import attr +from ..environment import MYPY_RUNNING from ..exceptions import InvalidPythonVersion from ..utils import ensure_path from .mixins import BaseFinder @@ -14,32 +15,31 @@ from .path import PathEntry from .python import PythonVersion, VersionMap +if MYPY_RUNNING: + from typing import DefaultDict, Tuple, List, Optional, Union, TypeVar, Type, Any + FinderType = TypeVar('FinderType') + + @attr.s class WindowsFinder(BaseFinder): - paths = attr.ib(default=attr.Factory(list)) - version_list = attr.ib(default=attr.Factory(list)) - versions = attr.ib() - pythons = attr.ib() + paths = attr.ib(default=attr.Factory(list), type=list) + version_list = attr.ib(default=attr.Factory(list), type=list) + _versions = attr.ib() # type: DefaultDict[Tuple, PathEntry] + _pythons = attr.ib() # type: DefaultDict[str, PathEntry] def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type (...) -> List[PathEntry] version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "matches", major, minor, patch, pre, dev, arch, python_version=name ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -49,33 +49,30 @@ class WindowsFinder(BaseFinder): def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): - return next( - ( - v - for v in self.find_all_python_versions( - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=None, - ) - ), - None, + # type: (...) -> Optional[PathEntry] + return next(iter(v for v in self.find_all_python_versions( + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=None, + )), None ) - @versions.default + @_versions.default def get_versions(self): - versions = defaultdict(PathEntry) + # type: () -> DefaultDict[Tuple, PathEntry] + versions = defaultdict(PathEntry) # type: DefaultDict[Tuple, PathEntry] from pythonfinder._vendor.pep514tools import environment as pep514env env_versions = pep514env.findall() @@ -92,25 +89,48 @@ class WindowsFinder(BaseFinder): py_version = PythonVersion.from_windows_launcher(version_object) except InvalidPythonVersion: continue + if py_version is None: + continue self.version_list.append(py_version) + python_path = py_version.comes_from.path if py_version.comes_from else py_version.executable + python_kwargs = {python_path: py_version} if python_path is not None else {} base_dir = PathEntry.create( path, is_root=True, only_python=True, - pythons={py_version.comes_from.path: py_version}, + pythons=python_kwargs, ) versions[py_version.version_tuple[:5]] = base_dir self.paths.append(base_dir) return versions - @pythons.default + @property + def versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] + if not self._versions: + self._versions = self.get_versions() + return self._versions + + @_pythons.default def get_pythons(self): - pythons = defaultdict() + # type: () -> DefaultDict[str, PathEntry] + pythons = defaultdict() # type: DefaultDict[str, PathEntry] for version in self.version_list: _path = ensure_path(version.comes_from.path) pythons[_path.as_posix()] = version.comes_from return pythons + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value + @classmethod - def create(cls): + def create(cls, *args, **kwargs): + # type: (Type[FinderType], Any, Any) -> FinderType return cls() diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 011754ea..edbc149b 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -1,26 +1,46 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import -import os -import six +from __future__ import absolute_import, print_function + import operator -from .models import SystemPath +import os + +import six + +from click import secho from vistir.compat import lru_cache +from . import environment +from .exceptions import InvalidPythonVersion +from .models import path +from .utils import Iterable, filter_pythons + + +if environment.MYPY_RUNNING: + from typing import Optional, Dict, Any, Union, List, Iterator + from .models.path import Path, PathEntry + from .models.windows import WindowsFinder + from .models.path import SystemPath + class Finder(object): - def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=True): - """ - Finder A cross-platform Finder for locating python and other executables. - Searches for python and other specified binaries starting in `path`, if supplied, - but searching the bin path of `sys.executable` if `system=True`, and then - searching in the `os.environ['PATH']` if `global_search=True`. When `global_search` - is `False`, this search operation is restricted to the allowed locations of - `path` and `system`. + """ + A cross-platform Finder for locating python and other executables. + + Searches for python and other specified binaries starting in *path*, if supplied, + but searching the bin path of ``sys.executable`` if *system* is ``True``, and then + searching in the ``os.environ['PATH']`` if *global_search* is ``True``. When *global_search* + is ``False``, this search operation is restricted to the allowed locations of + *path* and *system*. + """ + + def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=True): + # type: (Optional[str], bool, bool, bool) -> None + """Create a new :class:`~pythonfinder.pythonfinder.Finder` instance. :param path: A bin-directory search location, defaults to None :param path: str, optional - :param system: Whether to include the bin-dir of `sys.executable`, defaults to False + :param system: Whether to include the bin-dir of ``sys.executable``, defaults to False :param system: bool, optional :param global_search: Whether to search the global path from os.environ, defaults to True :param global_search: bool, optional @@ -29,34 +49,63 @@ class Finder(object): :returns: a :class:`~pythonfinder.pythonfinder.Finder` object. """ - self.path_prepend = path - self.global_search = global_search - self.system = system - self.ignore_unsupported = ignore_unsupported - self._system_path = None - self._windows_finder = None + self.path_prepend = path # type: Optional[str] + self.global_search = global_search # type: bool + self.system = system # type: bool + self.ignore_unsupported = ignore_unsupported # type: bool + self._system_path = None # type: Optional[SystemPath] + self._windows_finder = None # type: Optional[WindowsFinder] def __hash__(self): + # type: () -> int return hash( (self.path_prepend, self.system, self.global_search, self.ignore_unsupported) ) def __eq__(self, other): + # type: (Any) -> bool return self.__hash__() == other.__hash__() + def create_system_path(self): + # type: () -> SystemPath + return path.SystemPath.create( + path=self.path_prepend, system=self.system, global_search=self.global_search, + ignore_unsupported=self.ignore_unsupported + ) + + def reload_system_path(self): + # type: () -> None + """ + Rebuilds the base system path and all of the contained finders within it. + + This will re-apply any changes to the environment or any version changes on the system. + """ + + if self._system_path is not None: + self._system_path.clear_caches() + self._system_path = None + six.moves.reload_module(path) + self._system_path = self.create_system_path() + + def rehash(self): + # type: () -> None + if not self._system_path: + self._system_path = self.create_system_path() + self.find_all_python_versions.cache_clear() + self.find_python_version.cache_clear() + self.reload_system_path() + filter_pythons.cache_clear() + @property def system_path(self): - if not self._system_path: - self._system_path = SystemPath.create( - path=self.path_prepend, - system=self.system, - global_search=self.global_search, - ignore_unsupported=self.ignore_unsupported, - ) + # type: () -> SystemPath + if self._system_path is None: + self._system_path = self.create_system_path() return self._system_path @property def windows_finder(self): + # type: () -> Optional[WindowsFinder] if os.name == "nt" and not self._windows_finder: from .models import WindowsFinder @@ -64,13 +113,36 @@ class Finder(object): return self._windows_finder def which(self, exe): + # type: (str) -> Optional[PathEntry] return self.system_path.which(exe) @lru_cache(maxsize=1024) def find_python_version( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): + # type: (Optional[Union[str, int]], Optional[int], Optional[int], Optional[bool], Optional[bool], Optional[str], Optional[str]) -> PathEntry + """ + Find the python version which corresponds most closely to the version requested. + + :param Union[str, int] major: The major version to look for, or the full version, or the name of the target version. + :param Optional[int] minor: The minor version. If provided, disables string-based lookups from the major version field. + :param Optional[int] patch: The patch version. + :param Optional[bool] pre: If provided, specifies whether to search pre-releases. + :param Optional[bool] dev: If provided, whether to search dev-releases. + :param Optional[str] arch: If provided, which architecture to search. + :param Optional[str] name: *Name* of the target python, e.g. ``anaconda3-5.3.0`` + :return: A new *PathEntry* pointer at a matching python version, if one can be located. + :rtype: :class:`pythonfinder.models.path.PathEntry` + """ + from .models import PythonVersion + minor = int(minor) if minor is not None else minor + patch = int(patch) if patch is not None else patch + + version_dict = { + "minor": minor, + "patch": patch + } # type: Dict[str, Union[str, int, Any]] if ( isinstance(major, six.string_types) @@ -79,7 +151,7 @@ class Finder(object): and dev is None and patch is None ): - if arch is None and "-" in major: + if arch is None and "-" in major and major[0].isdigit(): orig_string = "{0!s}".format(major) major, _, arch = major.rpartition("-") if arch.startswith("x"): @@ -91,25 +163,43 @@ class Finder(object): arch = None else: arch = "{0}bit".format(arch) - try: - version_dict = PythonVersion.parse(major) - except ValueError: - if name is None: - name = "{0!s}".format(major) - major = None - version_dict = {} - major = version_dict.get("major", major) - minor = version_dict.get("minor", minor) - patch = version_dict.get("patch", patch) - pre = version_dict.get("is_prerelease", pre) if pre is None else pre - dev = version_dict.get("is_devrelease", dev) if dev is None else dev - arch = version_dict.get("architecture", arch) if arch is None else arch - if os.name == "nt": + try: + version_dict = PythonVersion.parse(major) + except (ValueError, InvalidPythonVersion): + if name is None: + name = "{0!s}".format(major) + major = None + version_dict = {} + elif major[0].isalpha(): + name = "%s" % major + major = None + else: + version_dict = { + "major": major, + "minor": minor, + "patch": patch, + "pre": pre, + "dev": dev, + "arch": arch + } + if version_dict.get("minor") is not None: + minor = int(version_dict["minor"]) + if version_dict.get("patch") is not None: + patch = int(version_dict["patch"]) + if version_dict.get("major") is not None: + major = int(version_dict["major"]) + _pre = version_dict.get("is_prerelease", pre) + pre = bool(_pre) if _pre is not None else pre + _dev = version_dict.get("is_devrelease", dev) + dev = bool(_dev) if _dev is not None else dev + arch = version_dict.get("architecture", None) if arch is None else arch # type: ignore + if os.name == "nt" and self.windows_finder is not None: match = self.windows_finder.find_python_version( major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) if match: return match + secho("Using name: %s" % name, fg="white") return self.system_path.find_python_version( major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) @@ -118,28 +208,26 @@ class Finder(object): def find_all_python_versions( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): + # type: (Optional[Union[str, int]], Optional[int], Optional[int], Optional[bool], Optional[bool], Optional[str], Optional[str]) -> List[PathEntry] version_sort = operator.attrgetter("as_python.version_sort") python_version_dict = getattr(self.system_path, "python_version_dict") if python_version_dict: - paths = filter( - None, - [ + paths = ( path for version in python_version_dict.values() for path in version - if path.as_python - ], + if path is not None and path.as_python ) - paths = sorted(paths, key=version_sort, reverse=True) - return paths + path_list = sorted(paths, key=version_sort, reverse=True) + return path_list versions = self.system_path.find_all_python_versions( major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) - if not isinstance(versions, list): - versions = [versions] - paths = sorted(versions, key=version_sort, reverse=True) - path_map = {} - for path in paths: + if not isinstance(versions, Iterable): + versions = [versions,] + path_list = sorted(versions, key=version_sort, reverse=True) + path_map = {} # type: Dict[str, PathEntry] + for path in path_list: try: resolved_path = path.path.resolve() except OSError: diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 18441919..24ffdea0 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -1,34 +1,39 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function +import io import itertools import os +import re from fnmatch import fnmatch import attr -import io -import re import six - import vistir from packaging.version import LegacyVersion, Version -from .environment import PYENV_ROOT, ASDF_DATA_DIR, MYPY_RUNNING +from .environment import MYPY_RUNNING, PYENV_ROOT from .exceptions import InvalidPythonVersion -six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) -from six.moves import Iterable + +six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) # type: ignore # noqa +six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) # type: ignore # noqa +from six.moves import Iterable # type: ignore # noqa +from six.moves import Sequence # type: ignore # noqa try: from functools import lru_cache except ImportError: - from backports.functools_lru_cache import lru_cache + from backports.functools_lru_cache import lru_cache # type: ignore # noqa if MYPY_RUNNING: - from typing import Any, Union, List, Callable, Iterable, Set, Tuple, Dict, Optional - from attr.validators import _OptionalValidator + from typing import ( + Any, Union, List, Callable, Iterable, Set, Tuple, Dict, Optional, Iterator + ) + from attr.validators import _OptionalValidator # type: ignore + from .models.path import PathEntry version_re = re.compile(r"(?P\d+)(?:\.(?P\d+))?(?:\.(?P(?<=\.)[0-9]+))?\.?" @@ -40,7 +45,12 @@ PYTHON_IMPLEMENTATIONS = ( "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", "stackless", "activepython", "micropython" ) -RULES_BASE = ["*{0}", "*{0}?", "*{0}?.?", "*{0}?.?m"] +RE_MATCHER = re.compile(r"(({0})(?:\d?(?:\.\d[cpm]{{0,3}}))?(?:-?[\d\.]+)*[^z])".format( + "|".join(PYTHON_IMPLEMENTATIONS) +)) +RULES_BASE = [ + "*{0}", "*{0}?", "*{0}?.?", "*{0}?.?m", "{0}?-?.?", "{0}?-?.?.?", "{0}?.?-?.?.?" +] RULES = [rule.format(impl) for impl in PYTHON_IMPLEMENTATIONS for rule in RULES_BASE] KNOWN_EXTS = {"exe", "py", "fish", "sh", ""} @@ -178,7 +188,10 @@ def looks_like_python(name): if not any(name.lower().startswith(py_name) for py_name in PYTHON_IMPLEMENTATIONS): return False - return any(fnmatch(name, rule) for rule in MATCH_RULES) + match = RE_MATCHER.match(name) + if match: + return any(fnmatch(name, rule) for rule in MATCH_RULES) + return False @lru_cache(maxsize=1024) @@ -198,7 +211,7 @@ def path_is_python(path): @lru_cache(maxsize=1024) def ensure_path(path): - # type: (Union[vistir.compat.Path, str]) -> bool + # type: (Union[vistir.compat.Path, str]) -> vistir.compat.Path """ Given a path (either a string or a Path object), expand variables and return a Path object. @@ -248,13 +261,16 @@ def unnest(item): item, target = itertools.tee(item, 2) else: target = item - for el in target: - if isinstance(el, Iterable) and not isinstance(el, six.string_types): - el, el_copy = itertools.tee(el, 2) - for sub in unnest(el_copy): - yield sub - else: - yield el + if getattr(target, "__iter__", None): + for el in target: + if isinstance(el, Iterable) and not isinstance(el, six.string_types): + el, el_copy = itertools.tee(el, 2) + for sub in unnest(el_copy): + yield sub + else: + yield el + else: + yield target def parse_pyenv_version_order(filename="version"): @@ -278,7 +294,8 @@ def parse_asdf_version_order(filename=".tool-versions"): line for line in contents.splitlines() if line.startswith("python") ), None) if python_section: - python_key, _, versions = python_section.partition(" ") + # python_key, _, versions + _, _, versions = python_section.partition(" ") if versions: return versions.split() return [] @@ -287,3 +304,35 @@ def parse_asdf_version_order(filename=".tool-versions"): # TODO: Reimplement in vistir def is_in_path(path, parent): return normalize_path(str(path)).startswith(normalize_path(str(parent))) + + +def expand_paths(path, only_python=True): + # type: (Union[Sequence, PathEntry], bool) -> Iterator + """ + Recursively expand a list or :class:`~pythonfinder.models.path.PathEntry` instance + + :param Union[Sequence, PathEntry] path: The path or list of paths to expand + :param bool only_python: Whether to filter to include only python paths, default True + :returns: An iterator over the expanded set of path entries + :rtype: Iterator[PathEntry] + """ + + if path is not None and (isinstance(path, Sequence) and + not getattr(path.__class__, "__name__", "") == "PathEntry"): + for p in unnest(path): + if p is None: + continue + for expanded in itertools.chain.from_iterable( + expand_paths(p, only_python=only_python) + ): + yield expanded + elif path is not None and path.is_dir: + for p in path.children.values(): + if p is not None and p.is_python and p.as_python is not None: + for sub_path in itertools.chain.from_iterable( + expand_paths(p, only_python=only_python) + ): + yield sub_path + else: + if path is not None and path.is_python and path.as_python is not None: + yield path