From ef060c4b9b5c119e60feaba167bb22a273e04d90 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 25 Jul 2018 17:02:45 -0400 Subject: [PATCH] Update pythonfinder Signed-off-by: Dan Ryan --- news/2582.bugfix | 1 + news/2582.feature | 4 + news/2582.vendor | 1 + pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/_vendor/Makefile | 14 ---- pipenv/vendor/pythonfinder/_vendor/vendor.txt | 1 - pipenv/vendor/pythonfinder/exceptions.py | 1 + pipenv/vendor/pythonfinder/models/__init__.py | 43 +++++++++-- pipenv/vendor/pythonfinder/models/path.py | 73 ++++++++++++------- pipenv/vendor/pythonfinder/models/pyenv.py | 4 +- pipenv/vendor/pythonfinder/models/python.py | 26 +++---- pipenv/vendor/pythonfinder/models/windows.py | 28 +++++-- pipenv/vendor/pythonfinder/pythonfinder.py | 48 ++++++++---- pipenv/vendor/pythonfinder/utils.py | 5 +- 14 files changed, 163 insertions(+), 88 deletions(-) create mode 100644 news/2582.bugfix create mode 100644 news/2582.feature create mode 100644 news/2582.vendor delete mode 100644 pipenv/vendor/pythonfinder/_vendor/Makefile delete mode 100644 pipenv/vendor/pythonfinder/_vendor/vendor.txt diff --git a/news/2582.bugfix b/news/2582.bugfix new file mode 100644 index 00000000..f434be81 --- /dev/null +++ b/news/2582.bugfix @@ -0,0 +1 @@ +Fixed multiple issues with finding the correct system python locations. diff --git a/news/2582.feature b/news/2582.feature new file mode 100644 index 00000000..4d1f7a36 --- /dev/null +++ b/news/2582.feature @@ -0,0 +1,4 @@ +Greatly enhanced python discovery functionality: + +- Added pep514 (windows launcher/finder) support for python discovery. +- Introduced architecture discovery for python installations which support different architectures. diff --git a/news/2582.vendor b/news/2582.vendor new file mode 100644 index 00000000..1a031eb7 --- /dev/null +++ b/news/2582.vendor @@ -0,0 +1 @@ +Update ``pythonfinder`` to major release ``1.0.0`` for integration. diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index eb3d0363..9f1628be 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = "0.1.4.dev0" +__version__ = "1.0.0" __all__ = ["Finder", "WindowsFinder", "SystemPath", "InvalidPythonVersion"] from .pythonfinder import Finder diff --git a/pipenv/vendor/pythonfinder/_vendor/Makefile b/pipenv/vendor/pythonfinder/_vendor/Makefile deleted file mode 100644 index 5c44fea4..00000000 --- a/pipenv/vendor/pythonfinder/_vendor/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -# Taken from pip: https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/src/pip/_vendor/Makefile -all: clean vendor - -clean: - @# Delete vendored items - find . -maxdepth 1 -mindepth 1 -type d -exec rm -rf {} \; - -vendor: - @# Install vendored libraries - pip install -t . -r vendor.txt - - @# Cleanup .egg-info directories - rm -rf *.egg-info - rm -rf *.dist-info diff --git a/pipenv/vendor/pythonfinder/_vendor/vendor.txt b/pipenv/vendor/pythonfinder/_vendor/vendor.txt deleted file mode 100644 index 88752498..00000000 --- a/pipenv/vendor/pythonfinder/_vendor/vendor.txt +++ /dev/null @@ -1 +0,0 @@ --e git+https://github.com/zooba/pep514tools.git@320e48745660b696e2dcaee888fc2e516b435e48#egg=pep514tools diff --git a/pipenv/vendor/pythonfinder/exceptions.py b/pipenv/vendor/pythonfinder/exceptions.py index 13e56e2c..df381daf 100644 --- a/pipenv/vendor/pythonfinder/exceptions.py +++ b/pipenv/vendor/pythonfinder/exceptions.py @@ -4,4 +4,5 @@ from __future__ import print_function, absolute_import class InvalidPythonVersion(Exception): """Raised when parsing an invalid python version""" + pass diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index 7cf0fadf..a38494ed 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -40,10 +40,19 @@ class BasePath(object): for ext in KNOWN_EXTS ] children = self.children - found = next((children[(self.path / child).as_posix()] for child in valid_names if (self.path / child).as_posix() in children), None) + found = next( + ( + children[(self.path / child).as_posix()] + for child in valid_names + if (self.path / child).as_posix() in children + ), + None, + ) return found - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -57,7 +66,9 @@ class BasePath(object): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - call_method = "find_all_python_versions" if self.is_dir else "find_python_version" + call_method = ( + "find_all_python_versions" if self.is_dir else "find_python_version" + ) sub_finder = operator.methodcaller( call_method, major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) @@ -67,7 +78,9 @@ class BasePath(object): version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_python_version( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -81,7 +94,13 @@ class BasePath(object): """ version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") @@ -89,13 +108,23 @@ class BasePath(object): if self.is_python and self.as_python and version_matcher(self.as_python): return self return - finder = ((child, child.as_python) for child in unnest(self.pythons.values()) if child.as_python) + finder = ( + (child, child.as_python) + for child in unnest(self.pythons.values()) + if child.as_python + ) py_filter = filter( None, filter(lambda child: version_matcher(child[1]), finder) ) version_sort = operator.attrgetter("version_sort") return next( - (c[0] for c in sorted(py_filter, key=lambda child: child[1].version_sort, reverse=True)), None + ( + c[0] + for c in sorted( + py_filter, key=lambda child: child[1].version_sort, reverse=True + ) + ), + None, ) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 6857d17d..af103915 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -49,7 +49,11 @@ class SystemPath(object): @cached_property def executables(self): - self.executables = [p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable] + self.executables = [ + p + for p in chain(*(child.children.values() for child in self.paths.values())) + if p.is_executable + ] return self.executables @cached_property @@ -69,7 +73,7 @@ class SystemPath(object): self._version_dict = defaultdict(list) for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): - if finder_name == 'windows': + if finder_name == "windows": if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue @@ -98,17 +102,15 @@ class SystemPath(object): self._setup_windows() if PYENV_INSTALLED: self._setup_pyenv() - venv = os.environ.get('VIRTUAL_ENV') - if os.name == 'nt': - bin_dir = 'Scripts' + venv = os.environ.get("VIRTUAL_ENV") + if os.name == "nt": + bin_dir = "Scripts" else: - bin_dir = 'bin' + bin_dir = "bin" if venv and (self.system or self.global_search): p = ensure_path(venv) self.path_order = [(p / bin_dir).as_posix()] + self.path_order - self.paths[p] = PathEntry.create( - path=p, is_root=True, only_python=False - ) + self.paths[p] = PathEntry.create(path=p, is_root=True, only_python=False) if self.system: syspath = Path(sys.executable) syspath_bin = syspath.parent @@ -141,7 +143,7 @@ class SystemPath(object): before_path + [p.path.as_posix() for p in root_paths] + after_path ) self.paths.update({p.path: p for p in root_paths}) - self._register_finder('pyenv', self.pyenv_finder) + self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): from .windows import WindowsFinder @@ -151,13 +153,13 @@ class SystemPath(object): path_addition = [p.path.as_posix() for p in root_paths] self.path_order = self.path_order[:] + path_addition self.paths.update({p.path: p for p in root_paths}) - self._register_finder('windows', self.windows_finder) + self._register_finder("windows", self.windows_finder) def get_path(self, path): path = ensure_path(path) _path = self.paths.get(path.as_posix()) if not _path and path.as_posix() in self.path_order: - _path = PathEntry.create( + _path = PathEntry.create( path=path.absolute(), is_root=True, only_python=self.only_python ) self.paths[path.as_posix()] = _path @@ -185,7 +187,9 @@ class SystemPath(object): filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) return next((f for f in filtered), None) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -200,18 +204,28 @@ class SystemPath(object): """ sub_finder = operator.methodcaller( - "find_all_python_versions", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "find_all_python_versions", + major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version paths = (self.get_path(k) for k in self.path_order) - path_filter = filter(None, unnest((sub_finder(p) for p in paths if p is not None))) + path_filter = filter( + None, unnest((sub_finder(p) for p in paths if p is not None)) + ) version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_python_version( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search for a specific python version on the path. :param major: Major python version to search for. @@ -226,7 +240,13 @@ class SystemPath(object): """ sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "find_python_version", + major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) if major and minor and patch: _tuple_pre = pre if pre is not None else False @@ -247,7 +267,7 @@ class SystemPath(object): if ver.as_python.version_tuple[:5] in self.python_version_dict: self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) else: - self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver,] + self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver] return ver @classmethod @@ -280,7 +300,13 @@ class SystemPath(object): for p in _path_objects } ) - return cls(paths=path_entries, path_order=paths, only_python=only_python, system=system, global_search=global_search) + return cls( + paths=path_entries, + path_order=paths, + only_python=only_python, + system=system, + global_search=global_search, + ) @attr.s @@ -293,7 +319,7 @@ class PathEntry(BasePath): pythons = attr.ib() def __str__(self): - return fs_str('{0}'.format(self.path.as_posix())) + return fs_str("{0}".format(self.path.as_posix())) def _filter_children(self): if self.only_python: @@ -333,6 +359,7 @@ class PathEntry(BasePath): if not self.py_version: try: from .python import PythonVersion + self.py_version = PythonVersion.from_path(self.path) except (ValueError, InvalidPythonVersion): self.py_version = None @@ -355,11 +382,7 @@ class PathEntry(BasePath): """ target = ensure_path(path) - creation_args = { - "path": target, - "is_root": is_root, - "only_python": only_python - } + creation_args = {"path": target, "is_root": is_root, "only_python": only_python} if pythons: creation_args["pythons"] = pythons _new = cls(**creation_args) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 13476b6b..6c890936 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -32,7 +32,9 @@ class PyenvFinder(BaseFinder): version.get("is_prerelease"), version.get("is_devrelease"), ) - versions[version_tuple] = VersionPath.create(path=p.resolve(), only_python=True) + versions[version_tuple] = VersionPath.create( + path=p.resolve(), only_python=True + ) return versions @pythons.default diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index fcb48723..6176fad4 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -44,13 +44,7 @@ class PythonVersion(object): release_sort = 1 elif self.is_devrelease: release_sort = 0 - return ( - self.major, - self.minor, - self.patch if self.patch else 0, - release_sort - ) - + return (self.major, self.minor, self.patch if self.patch else 0, release_sort) @property def version_tuple(self): @@ -68,9 +62,11 @@ class PythonVersion(object): self.is_devrelease, ) - def matches(self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None): - if arch and arch.isdigit(): - arch = '{0}bit'.format(arch) + def matches( + self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None + ): + if arch and arch.isnumeric(): + arch = "{0}bit".format(arch) return ( (major is None or self.major == major) and (minor is None or self.minor == minor) @@ -195,9 +191,9 @@ class PythonVersion(object): @classmethod def create(cls, **kwargs): - if 'architecture' in kwargs: - if kwargs['architecture'].isdigit(): - kwargs['architecture'] = '{0}bit'.format(kwargs['architecture']) + if "architecture" in kwargs: + if kwargs["architecture"].isnumeric(): + kwargs["architecture"] = "{0}bit".format(kwargs["architecture"]) return cls(**kwargs) @@ -221,4 +217,6 @@ class VersionMap(object): current_entries = {p.path for p in self.versions.get(version)} new_entries = {p.path for p in entries} new_entries -= current_entries - self.versions[version].append([e for e in entries if e.path in new_entries]) + self.versions[version].append( + [e for e in entries if e.path in new_entries] + ) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index 40dd93ed..f731432c 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -17,9 +17,17 @@ class WindowsFinder(BaseFinder): versions = attr.ib() pythons = attr.ib() - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -27,11 +35,17 @@ class WindowsFinder(BaseFinder): version_sort = operator.attrgetter("version_sort") return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): - return next(( - v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch - )), None + def find_python_version( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): + return next( + ( + v + for v in self.find_all_python_versions( + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + ) + ), + None, ) @versions.default diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index c6edbaf9..d6f49bf7 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -35,7 +35,9 @@ class Finder(object): def system_path(self): if not self._system_path: self._system_path = SystemPath.create( - path=self.path_prepend, system=self.system, global_search=self.global_search + path=self.path_prepend, + system=self.system, + global_search=self.global_search, ) return self._system_path @@ -50,12 +52,21 @@ class Finder(object): def which(self, exe): return self.system_path.which(exe) - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_python_version( + self, major, minor=None, patch=None, pre=None, dev=None, arch=None + ): from .models import PythonVersion - if isinstance(major, six.string_types) and pre is None and minor is None and dev is None and patch is None: - if arch is None and '-' in major: - major, arch = major.rsplit('-', 1) - if not arch.isdigit(): + + if ( + isinstance(major, six.string_types) + and pre is None + and minor is None + and dev is None + and patch is None + ): + if arch is None and "-" in major: + major, arch = major.rsplit("-", 1) + if not arch.isnumeric(): major = "{0}-{1}".format(major, arch) else: arch = "{0}bit".format(arch) @@ -76,19 +87,28 @@ class Finder(object): major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): version_sort = operator.attrgetter("as_python.version_sort") - python_version_dict = getattr(self.system_path, 'python_version_dict') + python_version_dict = getattr(self.system_path, "python_version_dict") if python_version_dict: - paths = filter(None, [path for version in python_version_dict.values() for path in version if path.as_python]) + paths = filter( + None, + [ + path + for version in python_version_dict.values() + for path in version + if path.as_python + ], + ) paths = sorted(paths, key=version_sort, reverse=True) return paths - versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) + versions = self.system_path.find_all_python_versions( + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + ) if not isinstance(versions, list): - versions = [versions,] - # if os.name == 'nt': - # windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) - # versions = list(windows_versions) + versions + versions = [versions] paths = sorted(versions, key=version_sort, reverse=True) path_map = {} for path in paths: diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index fdc54381..1a9bfa2c 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -32,10 +32,7 @@ def _run(cmd): """ encoding = locale.getdefaultlocale()[1] or "utf-8" c = subprocess.Popen( - cmd, - env=os.environ.copy(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + cmd, env=os.environ.copy(), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) out, err = c.communicate() return out.decode(encoding).strip(), err.decode(encoding).strip()