From 00dd84537244262e98db1fa45319d33254f383e7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 23 Jul 2018 12:38:50 -0400 Subject: [PATCH 001/101] Enable parsing of dependency links in setup.py - Fixed errors with url parsing during hashing Signed-off-by: Dan Ryan --- news/2434.bugfix | 1 + pipenv/patched/piptools/repositories/pypi.py | 14 ++++--- .../vendoring/patches/patched/piptools.patch | 38 +++++++++++-------- 3 files changed, 33 insertions(+), 20 deletions(-) create mode 100644 news/2434.bugfix diff --git a/news/2434.bugfix b/news/2434.bugfix new file mode 100644 index 00000000..0a9603e1 --- /dev/null +++ b/news/2434.bugfix @@ -0,0 +1 @@ +Fixed the ability of pipenv to parse ``dependency_links`` from ``setup.py`` when ``PIP_PROCESS_DEPENDENCY_LINKS`` is enabled. diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index f09ff372..2f746094 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,7 +1,7 @@ # coding: utf-8 from __future__ import (absolute_import, division, print_function, unicode_literals) - +import copy import hashlib import os import sys @@ -64,15 +64,19 @@ class HashCache(SafeFileCache): def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - can_hash = location.hash + vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') + new_location = copy.deepcopy(location) + if any(new_location.url.startswith(vcs) for vcs in vcs_uris): + new_location.url = new_location.url.split("+", 1)[-1] + can_hash = new_location.hash if can_hash: # hash url WITH fragment - hash_value = self.get(location.url) + hash_value = self.get(new_location.url) if not hash_value: - hash_value = self._get_file_hash(location) + hash_value = self._get_file_hash(new_location) hash_value = hash_value.encode('utf8') if can_hash: - self.set(location.url, hash_value) + self.set(new_location.url, hash_value) return hash_value.decode('utf8') def _get_file_hash(self, location): diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index f5fb822f..56696e08 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,11 +19,15 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..c922be1 100644 +index 1c4b943..245e9ce 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, - +@@ -1,9 +1,10 @@ + # coding: utf-8 + from __future__ import (absolute_import, division, print_function, + unicode_literals) +- ++import copy import hashlib import os +import sys @@ -58,7 +62,7 @@ index 1c4b943..c922be1 100644 from .base import BaseRepository -@@ -37,6 +49,40 @@ except ImportError: +@@ -37,6 +49,44 @@ except ImportError: from pip.wheel import WheelCache @@ -77,15 +81,19 @@ index 1c4b943..c922be1 100644 + def get_hash(self, location): + # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it + hash_value = None -+ can_hash = location.hash ++ vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') ++ new_location = copy.deepcopy(location) ++ if any(new_location.url.startswith(vcs) for vcs in vcs_uris): ++ new_location.url = new_location.url.split("+", 1)[-1] ++ can_hash = new_location.hash + if can_hash: + # hash url WITH fragment -+ hash_value = self.get(location.url) ++ hash_value = self.get(new_location.url) + if not hash_value: -+ hash_value = self._get_file_hash(location) ++ hash_value = self._get_file_hash(new_location) + hash_value = hash_value.encode('utf8') + if can_hash: -+ self.set(location.url, hash_value) ++ self.set(new_location.url, hash_value) + return hash_value.decode('utf8') + + def _get_file_hash(self, location): @@ -99,7 +107,7 @@ index 1c4b943..c922be1 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,10 +92,11 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +96,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -113,7 +121,7 @@ index 1c4b943..c922be1 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -74,11 +121,15 @@ class PyPIRepository(BaseRepository): +@@ -74,11 +125,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -131,7 +139,7 @@ index 1c4b943..c922be1 100644 def freshen_build_caches(self): """ -@@ -114,10 +165,14 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +169,14 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -148,7 +156,7 @@ index 1c4b943..c922be1 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +181,71 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +185,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -223,7 +231,7 @@ index 1c4b943..c922be1 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,20 +270,40 @@ class PyPIRepository(BaseRepository): +@@ -155,20 +274,40 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -268,7 +276,7 @@ index 1c4b943..c922be1 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +323,97 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +327,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -369,7 +377,7 @@ index 1c4b943..c922be1 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -217,24 +432,22 @@ class PyPIRepository(BaseRepository): +@@ -217,24 +436,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. From f3e4e73cff7ac91fc13da87f367b2bdbfb2b9d31 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 18:31:55 -0400 Subject: [PATCH 002/101] Fix resolution using `dependency_links` with ssh - Exclude VCS SSH uris from hashing - Add additional resilience to the piptools resolver - Fixes #2613 Signed-off-by: Dan Ryan --- news/2643.bugfix | 1 + pipenv/patched/piptools/repositories/pypi.py | 28 +++++---- .../vendoring/patches/patched/piptools.patch | 57 ++++++++++++------- 3 files changed, 55 insertions(+), 31 deletions(-) create mode 100644 news/2643.bugfix diff --git a/news/2643.bugfix b/news/2643.bugfix new file mode 100644 index 00000000..566879af --- /dev/null +++ b/news/2643.bugfix @@ -0,0 +1 @@ +Dependency links to private repositories defined via ``ssh://`` schemes will now install correctly and skip hashing as long as ``PIP_PROCESS_DEPENDENCY_LINKS=1``. diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 2f746094..feab21b5 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -21,18 +21,16 @@ from .._compat import ( SafeFileCache, ) -from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement -from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier -from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable -from pipenv.patched.notpip._vendor.pyparsing import ParseException +from pipenv.patched.notpip._vendor.packaging.requirements import Requirement +from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, Specifier +from pipenv.patched.notpip._vendor.packaging.markers import Op, Value, Variable from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.vcs import VcsSupport -from ..cache import CACHE_DIR from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound -from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, - make_install_requirement, format_requirement, dedup, clean_requires_python) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, + make_install_requirement, clean_requires_python) from .base import BaseRepository @@ -64,9 +62,10 @@ class HashCache(SafeFileCache): def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') + vcs = VcsSupport() + orig_scheme = location.scheme new_location = copy.deepcopy(location) - if any(new_location.url.startswith(vcs) for vcs in vcs_uris): + if orig_scheme in vcs.all_schemes: new_location.url = new_location.url.split("+", 1)[-1] can_hash = new_location.hash if can_hash: @@ -280,6 +279,11 @@ class PyPIRepository(BaseRepository): setup_requires = {} dist = None if ireq.editable: + try: + from setuptools.build_meta import _run_setup + _run_setup(ireq.setup_py) + except (ImportError, InstallationError): + pass try: dist = ireq.get_dist() except InstallationError: @@ -429,6 +433,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() + vcs = VcsSupport() + if ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + return set() + if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 56696e08..b527470b 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..245e9ce 100644 +index 1c4b943..9461709 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,9 +1,10 @@ @@ -34,7 +34,7 @@ index 1c4b943..245e9ce 100644 from contextlib import contextmanager from shutil import rmtree -@@ -15,13 +16,24 @@ from .._compat import ( +@@ -15,13 +16,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -44,25 +44,23 @@ index 1c4b943..245e9ce 100644 + SafeFileCache, ) -+from pip._vendor.packaging.requirements import InvalidRequirement, Requirement -+from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -+from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier -+from pip._vendor.packaging.markers import Marker, Op, Value, Variable -+from pip._vendor.pyparsing import ParseException +-from ..cache import CACHE_DIR ++from pip._vendor.packaging.requirements import Requirement ++from pip._vendor.packaging.specifiers import SpecifierSet, Specifier ++from pip._vendor.packaging.markers import Op, Value, Variable +from pip._internal.exceptions import InstallationError ++from pip._internal.vcs import VcsSupport + - from ..cache import CACHE_DIR +from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound --from ..utils import (fs_str, is_pinned_requirement, lookup_table, + from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) -+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, -+ make_install_requirement, format_requirement, dedup, clean_requires_python) ++ make_install_requirement, clean_requires_python) + from .base import BaseRepository -@@ -37,6 +49,44 @@ except ImportError: +@@ -37,6 +47,45 @@ except ImportError: from pip.wheel import WheelCache @@ -81,9 +79,10 @@ index 1c4b943..245e9ce 100644 + def get_hash(self, location): + # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it + hash_value = None -+ vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') ++ vcs = VcsSupport() ++ orig_scheme = location.scheme + new_location = copy.deepcopy(location) -+ if any(new_location.url.startswith(vcs) for vcs in vcs_uris): ++ if orig_scheme in vcs.all_schemes: + new_location.url = new_location.url.split("+", 1)[-1] + can_hash = new_location.hash + if can_hash: @@ -107,7 +106,7 @@ index 1c4b943..245e9ce 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,10 +96,11 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +95,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -121,7 +120,7 @@ index 1c4b943..245e9ce 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -74,11 +125,15 @@ class PyPIRepository(BaseRepository): +@@ -74,11 +124,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -139,7 +138,7 @@ index 1c4b943..245e9ce 100644 def freshen_build_caches(self): """ -@@ -114,10 +169,14 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +168,14 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -156,7 +155,7 @@ index 1c4b943..245e9ce 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +185,71 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +184,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -231,7 +230,7 @@ index 1c4b943..245e9ce 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,20 +274,40 @@ class PyPIRepository(BaseRepository): +@@ -155,20 +273,45 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -243,6 +242,11 @@ index 1c4b943..245e9ce 100644 + dist = None + if ireq.editable: + try: ++ from setuptools.build_meta import _run_setup ++ _run_setup(ireq.setup_py) ++ except (ImportError, InstallationError): ++ pass ++ try: + dist = ireq.get_dist() + except InstallationError: + ireq.run_egg_info() @@ -276,7 +280,7 @@ index 1c4b943..245e9ce 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +327,97 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +331,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -377,7 +381,18 @@ index 1c4b943..245e9ce 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -217,24 +436,22 @@ class PyPIRepository(BaseRepository): +@@ -210,6 +433,10 @@ class PyPIRepository(BaseRepository): + if ireq.editable: + return set() + ++ vcs = VcsSupport() ++ if ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: ++ return set() ++ + if not is_pinned_requirement(ireq): + raise TypeError( + "Expected pinned requirement, got {}".format(ireq)) +@@ -217,24 +444,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. From 5334183c436c6a1572645fbd7de7720eb51f419f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 19:11:36 -0400 Subject: [PATCH 003/101] Make sure there is a link to check against Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index feab21b5..daaa5902 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -434,7 +434,7 @@ class PyPIRepository(BaseRepository): return set() vcs = VcsSupport() - if ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: return set() if not is_pinned_requirement(ireq): diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index b527470b..8dd58066 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -386,7 +386,7 @@ index 1c4b943..9461709 100644 return set() + vcs = VcsSupport() -+ if ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: ++ if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + return set() + if not is_pinned_requirement(ireq): From 6e38560a7f8026caeeaa4a3b486115ff430a24c9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 19:35:44 -0400 Subject: [PATCH 004/101] Add chdir context manager and properly run setup.py files during resolution Signed-off-by: Dan Ryan --- news/2643.feature | 1 + pipenv/patched/piptools/repositories/pypi.py | 6 ++++-- pipenv/utils.py | 12 ++++++++++++ tasks/vendoring/patches/patched/piptools.patch | 16 +++++++++------- 4 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 news/2643.feature diff --git a/news/2643.feature b/news/2643.feature new file mode 100644 index 00000000..052398c7 --- /dev/null +++ b/news/2643.feature @@ -0,0 +1 @@ +Enhanced resolution of editable and VCS dependencies. diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index daaa5902..94879f2f 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -280,8 +280,10 @@ class PyPIRepository(BaseRepository): dist = None if ireq.editable: try: - from setuptools.build_meta import _run_setup - _run_setup(ireq.setup_py) + from pipenv.utils import chdir + with chdir(ireq.setup_py_dir): + from setuptools.dist import distutils + distutils.core.run_setup(ireq.setup_py) except (ImportError, InstallationError): pass try: diff --git a/pipenv/utils.py b/pipenv/utils.py index 08c08381..b45081fe 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1361,3 +1361,15 @@ def is_virtual_environment(path): if python_like.is_file() and os.access(str(python_like), os.X_OK): return True return False + + +@contextmanager +def chdir(path): + """Context manager to change working directories.""" + from ._compat import Path + prev_cwd = Path.cwd() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(prev_cwd) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 8dd58066..17f2e2b5 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..9461709 100644 +index 1c4b943..91902dc 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,9 +1,10 @@ @@ -230,7 +230,7 @@ index 1c4b943..9461709 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,20 +273,45 @@ class PyPIRepository(BaseRepository): +@@ -155,20 +273,47 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -242,8 +242,10 @@ index 1c4b943..9461709 100644 + dist = None + if ireq.editable: + try: -+ from setuptools.build_meta import _run_setup -+ _run_setup(ireq.setup_py) ++ from pipenv.utils import chdir ++ with chdir(ireq.setup_py_dir): ++ from setuptools.dist import distutils ++ distutils.core.run_setup(ireq.setup_py) + except (ImportError, InstallationError): + pass + try: @@ -280,7 +282,7 @@ index 1c4b943..9461709 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +331,97 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +333,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -381,7 +383,7 @@ index 1c4b943..9461709 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -210,6 +433,10 @@ class PyPIRepository(BaseRepository): +@@ -210,6 +435,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -392,7 +394,7 @@ index 1c4b943..9461709 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -217,24 +444,22 @@ class PyPIRepository(BaseRepository): +@@ -217,24 +446,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. From 1c6acfecb5db81575ea3af8266dbf08bb812324c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 20:24:06 -0400 Subject: [PATCH 005/101] Ensure that we use posix style strings instead of Path objects for chdir context manager Signed-off-by: Dan Ryan --- pipenv/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index b45081fe..be1240ac 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1367,7 +1367,9 @@ def is_virtual_environment(path): def chdir(path): """Context manager to change working directories.""" from ._compat import Path - prev_cwd = Path.cwd() + prev_cwd = Path.cwd().as_posix() + if isinstance(path, Path): + path = path.as_posix() os.chdir(str(path)) try: yield From c584739b417c52d06c4b14a0e58fba620e01dc00 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 20:31:27 -0400 Subject: [PATCH 006/101] Handle NoneType paths Signed-off-by: Dan Ryan --- pipenv/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipenv/utils.py b/pipenv/utils.py index be1240ac..286b1866 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1367,6 +1367,8 @@ def is_virtual_environment(path): def chdir(path): """Context manager to change working directories.""" from ._compat import Path + if not path: + return prev_cwd = Path.cwd().as_posix() if isinstance(path, Path): path = path.as_posix() From dc97d73e62bf89a8a57d9efb25c24676c0e39df3 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 23:37:28 -0400 Subject: [PATCH 007/101] Add exception handling for non-existent setup_py_dir Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 94879f2f..f92bd97e 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -284,7 +284,7 @@ class PyPIRepository(BaseRepository): with chdir(ireq.setup_py_dir): from setuptools.dist import distutils distutils.core.run_setup(ireq.setup_py) - except (ImportError, InstallationError): + except (ImportError, InstallationError, TypeError): pass try: dist = ireq.get_dist() diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 17f2e2b5..273955a1 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -246,7 +246,7 @@ index 1c4b943..91902dc 100644 + with chdir(ireq.setup_py_dir): + from setuptools.dist import distutils + distutils.core.run_setup(ireq.setup_py) -+ except (ImportError, InstallationError): ++ except (ImportError, InstallationError, TypeError): + pass + try: + dist = ireq.get_dist() From d33c4e86e18489e0ac51b1d225fe26e1d68d1d69 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 25 Jul 2018 00:14:44 -0400 Subject: [PATCH 008/101] Fix exception handling Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index f92bd97e..bf7ebd2d 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -284,7 +284,7 @@ class PyPIRepository(BaseRepository): with chdir(ireq.setup_py_dir): from setuptools.dist import distutils distutils.core.run_setup(ireq.setup_py) - except (ImportError, InstallationError, TypeError): + except (ImportError, InstallationError, TypeError, AttributeError): pass try: dist = ireq.get_dist() diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 273955a1..7d9b64d8 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -246,7 +246,7 @@ index 1c4b943..91902dc 100644 + with chdir(ireq.setup_py_dir): + from setuptools.dist import distutils + distutils.core.run_setup(ireq.setup_py) -+ except (ImportError, InstallationError, TypeError): ++ except (ImportError, InstallationError, TypeError, AttributeError): + pass + try: + dist = ireq.get_dist() From 0d2ceb63128625f5f552d5ac8e2f7b0ab0370ac3 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 18:43:07 +0800 Subject: [PATCH 009/101] Pull in PythonFinder updates --- pipenv/vendor/pythonfinder/models/path.py | 18 +++++++++++---- pipenv/vendor/pythonfinder/pythonfinder.py | 27 ++++++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 2c6ea950..63c8abdc 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -26,6 +26,7 @@ except ImportError: @attr.s class SystemPath(object): + global_search = attr.ib(default=True) paths = attr.ib(default=attr.Factory(defaultdict)) _executables = attr.ib(default=attr.Factory(list)) _python_executables = attr.ib(default=attr.Factory(list)) @@ -94,7 +95,10 @@ class SystemPath(object): (p for p in reversed(self.path_order) if PYENV_ROOT.lower() in p.lower()), None, ) - pyenv_index = self.path_order.index(last_pyenv) + try: + pyenv_index = self.path_order.index(last_pyenv) + except ValueError: + return self.pyenv_finder = PyenvFinder.create(root=PYENV_ROOT) # paths = (v.paths.values() for v in self.pyenv_finder.versions.values()) root_paths = ( @@ -200,7 +204,7 @@ class SystemPath(object): ) @classmethod - def create(cls, path=None, system=False, only_python=False): + def create(cls, path=None, system=False, only_python=False, global_search=True): """Create a new :class:`pythonfinder.models.SystemPath` instance. :param path: Search path to prepend when searching, defaults to None @@ -214,7 +218,9 @@ class SystemPath(object): """ path_entries = defaultdict(PathEntry) - paths = os.environ.get("PATH").split(os.pathsep) + paths = [] + if global_search: + paths = os.environ.get("PATH").split(os.pathsep) if path: paths = [path] + paths _path_objects = [ensure_path(p.strip('"')) for p in paths] @@ -227,7 +233,11 @@ class SystemPath(object): for p in _path_objects } ) - return cls(paths=path_entries, path_order=paths, only_python=only_python, system=system) + return cls( + paths=path_entries, path_order=paths, + only_python=only_python, + system=system, global_search=global_search, + ) @attr.s diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 50e1b8d8..7e5813bb 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -6,8 +6,30 @@ from .models import SystemPath class Finder(object): - def __init__(self, path=None, system=False): + def __init__(self, path=None, system=False, global_search=True): + """Cross-platform Finder for locating python and other executables. + + Searches for python and other specified binaries starting in `path`, + if supplied, but searching the bin path of `sys.executable` if + `system=True`, and then searching in the `os.environ['PATH']` if + `global_search=True`. + + When `global_search` is `False`, this search operation is restricted to + the allowed locations of `path` and `system`. + + :param path: A bin-directory search location, or None to ignore. + :type path: str or None + :param system: Whether to include the bin-dir of `sys.executable`, + defaults to False + :type system: bool + :param global_search: Whether to search the global path from + os.environ, defaults to True. + :type global_search: bool + :returns: a :class:`~pythonfinder.pythonfinder.Finder` object. + """ + self.path_prepend = path + self.global_search = global_search self.system = system self._system_path = None self._windows_finder = None @@ -16,7 +38,8 @@ class Finder(object): def system_path(self): if not self._system_path: self._system_path = SystemPath.create( - path=self.path_prepend, system=self.system + path=self.path_prepend, system=self.system, + global_search=self.global_search, ) return self._system_path From 6d7b26ae6c40379f91745d5ef6da868c739c0469 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 19:08:51 +0800 Subject: [PATCH 010/101] Minimal Pythonfinder intergration --- pipenv/core.py | 110 ++++--------------------------------------------- 1 file changed, 9 insertions(+), 101 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 25bd0b7e..40b2ef40 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -317,76 +317,15 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): project.write_toml(p) -def find_python_from_py(python): - """Find a Python executable from on Windows. - - Ask py.exe for its opinion. - """ - py = system_which("py") - if not py: - return None - - version_args = ["-{0}".format(python[0])] - if len(python) >= 2: - version_args.append("-{0}.{1}".format(python[0], python[2])) - import subprocess - - for ver_arg in reversed(version_args): - try: - python_exe = subprocess.check_output( - [py, ver_arg, "-c", "import sys; print(sys.executable)"] - ) - except subprocess.CalledProcessError: - continue - - if not isinstance(python_exe, str): - python_exe = python_exe.decode(sys.getdefaultencoding()) - python_exe = python_exe.strip() - version = python_version(python_exe) - if (version or "").startswith(python): - return python_exe - - -def find_python_in_path(python): - """Find a Python executable from a version number. - - This uses the PATH environment variable to locate an appropriate Python. - """ - possibilities = ["python", "python{0}".format(python[0])] - if len(python) >= 2: - possibilities.extend( - [ - "python{0}{1}".format(python[0], python[2]), - "python{0}.{1}".format(python[0], python[2]), - "python{0}.{1}m".format(python[0], python[2]), - ] - ) - # Reverse the list, so we find specific ones first. - possibilities = reversed(possibilities) - for possibility in possibilities: - # Windows compatibility. - if os.name == "nt": - possibility = "{0}.exe".format(possibility) - pythons = system_which(possibility, mult=True) - for p in pythons: - version = python_version(p) - if (version or "").startswith(python): - return p - - def find_a_system_python(python): - """Finds a system python, given a version (e.g. 2 / 2.7 / 3.6.2), or a full path.""" - if python.startswith("py"): - return system_which(python) - - elif os.path.isabs(python): - return python - - python_from_py = find_python_from_py(python) - if python_from_py: - return python_from_py - - return find_python_in_path(python) + if not python: + return None + from .vendor import pythonfinder + finder = pythonfinder.Finder() + python_entry = finder.find_python_version(python) + if python_entry: + return str(python_entry.path) + return None def ensure_python(three=None, python=None): @@ -409,35 +348,7 @@ def ensure_python(three=None, python=None): ) sys.exit(1) - def activate_pyenv(): - from notpip._vendor.packaging.version import parse as parse_version - - """Adds all pyenv installations to the PATH.""" - if PYENV_INSTALLED: - if PYENV_ROOT: - pyenv_paths = {} - for found in glob("{0}{1}versions{1}*".format(PYENV_ROOT, os.sep)): - pyenv_paths[os.path.split(found)[1]] = "{0}{1}bin".format( - found, os.sep - ) - for version_str, pyenv_path in pyenv_paths.items(): - version = parse_version(version_str) - if version.is_prerelease and pyenv_paths.get(version.base_version): - continue - - add_to_path(pyenv_path) - else: - click.echo( - "{0}: PYENV_ROOT is not set. New python paths will " - "probably not be exported properly after installation." - "".format(crayons.red("Warning", bold=True)), - err=True, - ) - global USING_DEFAULT_PYTHON - # Add pyenv paths to PATH. - activate_pyenv() - path_to_python = None USING_DEFAULT_PYTHON = three is None and not python # Find out which python is desired. if not python: @@ -446,8 +357,7 @@ def ensure_python(three=None, python=None): python = project.required_python_version if not python: python = PIPENV_DEFAULT_PYTHON_VERSION - if python: - path_to_python = find_a_system_python(python) + path_to_python = find_a_system_python(python) if not path_to_python and python is not None: # We need to install Python. click.echo( @@ -501,8 +411,6 @@ def ensure_python(three=None, python=None): click.echo(crayons.blue(e.err), err=True) # Print the results, in a beautiful blue… click.echo(crayons.blue(c.out), err=True) - # Add new paths to PATH. - activate_pyenv() # Find the newly installed Python, hopefully. version = str(version) path_to_python = find_a_system_python(version) From ac41cdcf4e3fe6d351f9dfe635cc90ddd4421535 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 19:17:23 +0800 Subject: [PATCH 011/101] Remove duplicate and unused code --- pipenv/core.py | 4 +--- pipenv/environments.py | 7 ------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 40b2ef40..13793331 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -6,7 +6,6 @@ import sys import shutil import time import tempfile -from glob import glob import json as simplejson import click import click_completion @@ -51,8 +50,6 @@ from .environments import ( PIPENV_SKIP_VALIDATION, PIPENV_HIDE_EMOJIS, PIPENV_INSTALL_TIMEOUT, - PYENV_ROOT, - PYENV_INSTALLED, PIPENV_YES, PIPENV_DONT_LOAD_ENV, PIPENV_DEFAULT_PYTHON_VERSION, @@ -369,6 +366,7 @@ def ensure_python(three=None, python=None): err=True, ) # Pyenv is installed + from .vendor.pythonfinder.environment import PYENV_INSTALLED if not PYENV_INSTALLED: abort() else: diff --git a/pipenv/environments.py b/pipenv/environments.py index afe82988..037e75a3 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -209,12 +209,5 @@ PIPENV_SKIP_VALIDATION = True # Internal, the default shell to use if shell detection fails. PIPENV_SHELL = os.environ.get("SHELL") or os.environ.get("PYENV_SHELL") -# Internal, to tell if pyenv is installed. -PYENV_ROOT = os.environ.get("PYENV_ROOT", os.path.expanduser("~/.pyenv")) -PYENV_INSTALLED = ( - bool(os.environ.get("PYENV_SHELL")) or - bool(os.environ.get("PYENV_ROOT")) -) - # Internal, to tell whether the command line session is interactive. SESSION_IS_INTERACTIVE = bool(os.isatty(sys.stdout.fileno())) From 59b974bfc19131b07daa7de131ab9aa644e4e180 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 19:25:04 +0800 Subject: [PATCH 012/101] Use Pythonfinder in help --- pipenv/help.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pipenv/help.py b/pipenv/help.py index 32ffa7c0..42a4df3f 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -5,7 +5,7 @@ import pipenv from pprint import pprint from .__version__ import __version__ -from .core import project, system_which, find_python_in_path, python_version +from .core import project, system_which, find_a_system_python, python_version from .pep508checker import lookup @@ -27,14 +27,10 @@ def get_pipenv_diagnostics(): print("") print("Other Python installations in `PATH`:") print("") - for python_v in ("2.5", "2.6", "2.7", "3.4", "3.5", "3.6", "3.7"): - found = find_python_in_path(python_v) - if found: - print(" - `{0}`: `{1}`".format(python_v, found)) - found = system_which("python{0}".format(python_v), mult=True) - if found: - for f in found: - print(" - `{0}`: `{1}`".format(python_v, f)) + for python_v in ("2.6", "2.7", "3.4", "3.5", "3.6", "3.7"): + entry = find_a_system_python(python_v) + if entry: + print(" - `{0}`: `{1}`".format(python_v, entry.path)) print("") for p in ("python", "python2", "python3", "py"): found = system_which(p, mult=True) From d5cebedee8146cd36acd8025f29571ace9f0dd79 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 19:29:05 +0800 Subject: [PATCH 013/101] Keep logic of using actual Python commands --- pipenv/core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pipenv/core.py b/pipenv/core.py index 13793331..b059ddc9 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -317,6 +317,10 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): def find_a_system_python(python): if not python: return None + if python.startswith("py"): + return system_which(python) + if os.path.isabs(python): + return python from .vendor import pythonfinder finder = pythonfinder.Finder() python_entry = finder.find_python_version(python) From b82d5587e369ab2da0894d9c6e654aa38f175fdc Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 19:43:24 +0800 Subject: [PATCH 014/101] Tape directly into Pythonfinder instead --- pipenv/help.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pipenv/help.py b/pipenv/help.py index 42a4df3f..b6e6b084 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -5,8 +5,9 @@ import pipenv from pprint import pprint from .__version__ import __version__ -from .core import project, system_which, find_a_system_python, python_version +from .core import project, system_which, python_version from .pep508checker import lookup +from .vendor import pythonfinder def print_utf(line): @@ -27,8 +28,9 @@ def get_pipenv_diagnostics(): print("") print("Other Python installations in `PATH`:") print("") + finder = pythonfinder.Finder() for python_v in ("2.6", "2.7", "3.4", "3.5", "3.6", "3.7"): - entry = find_a_system_python(python_v) + entry = finder.find_python_version(python_v) if entry: print(" - `{0}`: `{1}`".format(python_v, entry.path)) print("") From d3d5d563e7ab9cba1f7b4220b6111252ecd7c941 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 16:27:51 -0400 Subject: [PATCH 015/101] Update pythonfinder and make full use of it in support calls Signed-off-by: Dan Ryan --- pipenv/help.py | 21 +++++++------ pipenv/vendor/pythonfinder/models/path.py | 8 ++--- pipenv/vendor/pythonfinder/pythonfinder.py | 35 ++++++++++------------ 3 files changed, 27 insertions(+), 37 deletions(-) diff --git a/pipenv/help.py b/pipenv/help.py index b6e6b084..118a2272 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -3,9 +3,10 @@ import os import sys import pipenv +from itertools import chain from pprint import pprint from .__version__ import __version__ -from .core import project, system_which, python_version +from .core import project, system_which from .pep508checker import lookup from .vendor import pythonfinder @@ -28,16 +29,14 @@ def get_pipenv_diagnostics(): print("") print("Other Python installations in `PATH`:") print("") - finder = pythonfinder.Finder() - for python_v in ("2.6", "2.7", "3.4", "3.5", "3.6", "3.7"): - entry = finder.find_python_version(python_v) - if entry: - print(" - `{0}`: `{1}`".format(python_v, entry.path)) - print("") - for p in ("python", "python2", "python3", "py"): - found = system_which(p, mult=True) - for f in found: - print(" - `{0}`: `{1}`".format(python_version(f), f)) + finder = pythonfinder.Finder(system=False, global_search=True) + python_versions = (getattr(finder, 'system_path').find_all_python_versions(major) for major in (2, 3)) + python_paths = list(chain(*python_versions)) + for python in python_paths: + python_version = python.py_version.version + python_path = python.path.as_posix() + print(" - `{0}`: `{1}`".format(python_version, python_path)) + print("") print("PEP 508 Information:") print("") diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 63c8abdc..c394fb22 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -72,7 +72,7 @@ class SystemPath(object): bin_dir = 'Scripts' else: bin_dir = 'bin' - if venv: + if venv and (self.system or self.global_search): p = Path(venv) self.path_order = [(p / bin_dir).as_posix()] + self.path_order self.paths[p] = PathEntry.create( @@ -233,11 +233,7 @@ class SystemPath(object): for p in _path_objects } ) - return cls( - paths=path_entries, path_order=paths, - only_python=only_python, - system=system, global_search=global_search, - ) + return cls(paths=path_entries, path_order=paths, only_python=only_python, system=system, global_search=global_search) @attr.s diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 7e5813bb..32e62b18 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -7,24 +7,20 @@ from .models import SystemPath class Finder(object): def __init__(self, path=None, system=False, global_search=True): - """Cross-platform Finder for locating python and other executables. - - Searches for python and other specified binaries starting in `path`, - if supplied, but searching the bin path of `sys.executable` if - `system=True`, and then searching in the `os.environ['PATH']` if - `global_search=True`. - - When `global_search` is `False`, this search operation is restricted to - the allowed locations of `path` and `system`. - - :param path: A bin-directory search location, or None to ignore. - :type path: str or None - :param system: Whether to include the bin-dir of `sys.executable`, - defaults to False - :type system: bool - :param global_search: Whether to search the global path from - os.environ, defaults to True. - :type global_search: bool + """Finder A cross-platform Finder for locating python and other executables. + + Searches for python and other specified binaries starting in `path`, if supplied, + but searching the bin path of `sys.executable` if `system=True`, and then + searching in the `os.environ['PATH']` if `global_search=True`. When `global_search` + is `False`, this search operation is restricted to the allowed locations of + `path` and `system`. + + :param path: A bin-directory search location, defaults to None + :param path: str, optional + :param system: Whether to include the bin-dir of `sys.executable`, defaults to False + :param system: bool, optional + :param global_search: Whether to search the global path from os.environ, defaults to True + :param global_search: bool, optional :returns: a :class:`~pythonfinder.pythonfinder.Finder` object. """ @@ -38,8 +34,7 @@ class Finder(object): def system_path(self): if not self._system_path: self._system_path = SystemPath.create( - path=self.path_prepend, system=self.system, - global_search=self.global_search, + path=self.path_prepend, system=self.system, global_search=self.global_search ) return self._system_path From 33328c435ab028faae36709cbeb930ee5602d774 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 17:00:25 -0400 Subject: [PATCH 016/101] Fully integrate pythonfinder for system pythons Signed-off-by: Dan Ryan --- pipenv/core.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index b059ddc9..38cb5b7b 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -315,17 +315,24 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): def find_a_system_python(python): + from .vendor.pythonfinder import Finder + # system always refers to sys.executable, which could point at a virtualenv + # for global searches we most likely want to turn that off + finder = Finder(system=False, global_search=True) if not python: return None + # when using the python launcher on windows we can find the versions ourselves + if os.name == 'nt' and python.startswith("py -"): + python = python[len("py -"):] if python.startswith("py"): - return system_which(python) + python_entry = finder.which(python) + if python_entry: + return python_entry.path.as_posix() if os.path.isabs(python): return python - from .vendor import pythonfinder - finder = pythonfinder.Finder() python_entry = finder.find_python_version(python) if python_entry: - return str(python_entry.path) + return python_entry.path.as_posix() return None From aeb07cdec03d20dcf98914aa124510049a7615c4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 17:40:29 -0400 Subject: [PATCH 017/101] Fix search path Signed-off-by: Dan Ryan --- pipenv/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 38cb5b7b..1a002dcd 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -318,7 +318,7 @@ def find_a_system_python(python): from .vendor.pythonfinder import Finder # system always refers to sys.executable, which could point at a virtualenv # for global searches we most likely want to turn that off - finder = Finder(system=False, global_search=True) + finder = Finder(system=True, global_search=True) if not python: return None # when using the python launcher on windows we can find the versions ourselves @@ -330,6 +330,7 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python + python_entry = finder.find_python_version(python) if python_entry: return python_entry.path.as_posix() From 26c34641d148369cb4fad8b31d1c5f5da8251543 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 17:54:17 -0400 Subject: [PATCH 018/101] Updated buildkite to echo out the path Signed-off-by: Dan Ryan --- run-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/run-tests.sh b/run-tests.sh index b71fd4fb..692a69ea 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -17,6 +17,7 @@ fi export PATH="$HOME/.local/bin:$PATH" # pip uninstall -y pipenv +echo "Path: $PATH" echo "Installing Pipenv…" pip install -e "$(pwd)" --upgrade pipenv install --deploy --dev From 5da85d34b3981e161e6bee65197453a6c11b930b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 17:58:39 -0400 Subject: [PATCH 019/101] Add some debugging Signed-off-by: Dan Ryan --- pipenv/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 1a002dcd..4f7a08a2 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -330,7 +330,7 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - + python_entry = finder.find_python_version(python) if python_entry: return python_entry.path.as_posix() @@ -367,6 +367,7 @@ def ensure_python(three=None, python=None): if not python: python = PIPENV_DEFAULT_PYTHON_VERSION path_to_python = find_a_system_python(python) + click.echo("Found path to python: %s" % path_to_python) if not path_to_python and python is not None: # We need to install Python. click.echo( From c6a3284d307b973059071071b6aba637688fdb16 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 18:07:31 -0400 Subject: [PATCH 020/101] Updated buildkite to set home properly Signed-off-by: Dan Ryan --- run-tests.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/run-tests.sh b/run-tests.sh index 692a69ea..1243e033 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -15,7 +15,15 @@ if [[ ! -z "$TEST_SUITE" ]]; then echo "Using TEST_SUITE=$TEST_SUITE" fi -export PATH="$HOME/.local/bin:$PATH" +HOME=$(readlink -f ~/) +if [[ -z "$HOME" ]]; then + if [[ "$USER" == "root" ]]; then + HOME="/root" + fi +fi +if [[ ! -z "$HOME" ]]; then + export PATH="${HOME}/.local/bin:${PATH}" +fi # pip uninstall -y pipenv echo "Path: $PATH" echo "Installing Pipenv…" From dd06342428046978e685b03a5ace1b732af153cb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 18:13:37 -0400 Subject: [PATCH 021/101] More debug info for buildkite Signed-off-by: Dan Ryan --- pipenv/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 4f7a08a2..a488c72e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -330,7 +330,10 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - + click.echo("Path order: %s" % finder.system_path.path_order) + version = python.split('.')[0] + versions_avail = [v.path.as_posix() for v in finder.system_path.find_all_python_versions(version)] + click.echo("Python versions found: %s" % versions_avail) python_entry = finder.find_python_version(python) if python_entry: return python_entry.path.as_posix() From 684475e2c60ee9a4ba6304559c6a0c3fc0b08876 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 19:41:32 -0400 Subject: [PATCH 022/101] More debugging Signed-off-by: Dan Ryan --- pipenv/core.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index a488c72e..0658a096 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -316,6 +316,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): def find_a_system_python(python): from .vendor.pythonfinder import Finder + from itertools import chain # system always refers to sys.executable, which could point at a virtualenv # for global searches we most likely want to turn that off finder = Finder(system=True, global_search=True) @@ -332,7 +333,12 @@ def find_a_system_python(python): return python click.echo("Path order: %s" % finder.system_path.path_order) version = python.split('.')[0] - versions_avail = [v.path.as_posix() for v in finder.system_path.find_all_python_versions(version)] + python_versions = (getattr(finder, 'system_path').find_all_python_versions(major) for major in (2, 3)) + python_paths = list(chain(*python_versions)) + for python in python_paths: + python_version = python.py_version.version + python_path = python.path.as_posix() + click.echo(" - `{0}`: `{1}`".format(python_version, python_path)) click.echo("Python versions found: %s" % versions_avail) python_entry = finder.find_python_version(python) if python_entry: From c6fb1ea31f4099f78c71d5a818a46f818f5f980e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 19:59:27 -0400 Subject: [PATCH 023/101] Fix path search to include prereleases Signed-off-by: Dan Ryan --- pipenv/core.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 0658a096..23982adb 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -316,7 +316,6 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): def find_a_system_python(python): from .vendor.pythonfinder import Finder - from itertools import chain # system always refers to sys.executable, which could point at a virtualenv # for global searches we most likely want to turn that off finder = Finder(system=True, global_search=True) @@ -331,16 +330,9 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - click.echo("Path order: %s" % finder.system_path.path_order) - version = python.split('.')[0] - python_versions = (getattr(finder, 'system_path').find_all_python_versions(major) for major in (2, 3)) - python_paths = list(chain(*python_versions)) - for python in python_paths: - python_version = python.py_version.version - python_path = python.path.as_posix() - click.echo(" - `{0}`: `{1}`".format(python_version, python_path)) - click.echo("Python versions found: %s" % versions_avail) python_entry = finder.find_python_version(python) + if not python_entry: + python_entry = finder.find_python_version(python, pre=True) if python_entry: return python_entry.path.as_posix() return None From 5366253f39c70cfc44e57683deacc2427dbfb832 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 20:19:17 -0400 Subject: [PATCH 024/101] More robust path searching Signed-off-by: Dan Ryan --- pipenv/core.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 23982adb..c55cc734 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -12,6 +12,7 @@ import click_completion import crayons import dotenv import delegator +from first import first import pipfile from blindspin import spinner import six @@ -319,6 +320,7 @@ def find_a_system_python(python): # system always refers to sys.executable, which could point at a virtualenv # for global searches we most likely want to turn that off finder = Finder(system=True, global_search=True) + python_entry = None if not python: return None # when using the python launcher on windows we can find the versions ourselves @@ -330,14 +332,32 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - python_entry = finder.find_python_version(python) + version = [int(v) for v in python.split('.')] + minor = None + patch = None + if len(version) > 1: + minor = version[1] + if len(version) > 2: + patch = version[2] + python_entries = getattr(finder, 'system_path').find_all_python_versions(version[0]) + python_entries = [(entry.py_version.version_tuple, entry) for entry in python_entries] + if minor and patch: + _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor, patch)) + python_entry = _py[1] if _py else None + elif minor: + _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor)) + python_entry = _py[1] if _py else None + else: + _py = first(python_entries) + python_entry = _py[1] if _py else None if not python_entry: - python_entry = finder.find_python_version(python, pre=True) + version_str = "{0}.{1}".format(version[0], minor) if minor else "{0}".format(version[0]) + exe_name = "python{0}".format(version_str) + python_entry = finder.which(exe_name) if python_entry: return python_entry.path.as_posix() return None - def ensure_python(three=None, python=None): # Support for the PIPENV_PYTHON environment variable. from .environments import PIPENV_PYTHON From 50aff8e330893db5a1e6ac9fe34343b405bfb5a3 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 01:38:09 -0400 Subject: [PATCH 025/101] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/__init__.py | 10 +++---- pipenv/vendor/pythonfinder/models/path.py | 8 ++--- pipenv/vendor/pythonfinder/models/python.py | 30 +++++++++++++++++-- pipenv/vendor/pythonfinder/pythonfinder.py | 29 ++++-------------- 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index e455ffb0..0c6f0134 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -42,7 +42,7 @@ class BasePath(object): found = next((children[(self.path / child).as_posix()] for child in valid_names if (self.path / child).as_posix() in children), None) return found - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -55,21 +55,21 @@ class BasePath(object): """ version_matcher = operator.methodcaller( - "matches", major, minor=minor, patch=patch, pre=pre, dev=dev + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") if not self.is_dir: - if self.is_python and self.as_python.matches(major, minor=minor, patch=patch, pre=pre, dev=dev): + if self.is_python and self.as_python.matches(major=major, minor=minor, patch=patch, pre=pre, dev=dev): return self return finder = ((child, child.as_python) for child in self.children.values() if child.is_python and child.as_python) py_filter = filter( None, filter(lambda child: version_matcher(child[1]), finder) ) - version_sort = operator.attrgetter("version") + version_sort = operator.attrgetter("version_sort") return next( - (c[0] for c in sorted(py_filter, key=lambda child: child[1].version, reverse=True)), None + (c[0] for c in sorted(py_filter, key=lambda child: child[1].version_sort, reverse=True)), None ) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index c394fb22..893b6af0 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -151,7 +151,7 @@ class SystemPath(object): filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) return next((f for f in filtered), None) - def find_all_python_versions(self, major, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -173,10 +173,10 @@ class SystemPath(object): return windows_finder_version paths = (self.get_path(k) for k in self.path_order) path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) - version_sort = operator.attrgetter("as_python.version") + version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): """Search for a specific python version on the path. :param major: Major python version to search for. @@ -198,7 +198,7 @@ class SystemPath(object): return windows_finder_version paths = (self.get_path(k) for k in self.path_order) path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) - version_sort = operator.attrgetter("as_python.version") + version_sort = operator.attrgetter("as_python.version_sort") return next( (c for c in sorted(path_filter, key=version_sort, reverse=True)), None ) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 1f681767..4ff364fa 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -27,6 +27,30 @@ class PythonVersion(object): comes_from = attr.ib(default=None) executable = attr.ib(default=None) + @property + def version_sort(self): + """version_sort tuple for sorting against other instances of the same class. + + Returns a tuple of the python version but includes a point for non-dev, + and a point for non-prerelease versions. So released versions will have 2 points + for this value. E.g. `(3, 6, 6, 2)` is a release, `(3, 6, 6, 1)` is a prerelease, + `(3, 6, 6, 0)` is a dev release, and `(3, 6, 6, 3)` is a postrelease. + """ + release_sort = 2 + if self.is_postrelease: + release_sort = 3 + elif self.is_prerelease: + release_sort = 1 + elif self.is_devrelease: + release_sort = 0 + return ( + self.major, + self.minor, + self.patch, + release_sort + ) + + @property def version_tuple(self): """Provides a version tuple for using as a dictionary key. @@ -43,9 +67,9 @@ class PythonVersion(object): self.is_devrelease, ) - def matches(self, major, minor=None, patch=None, pre=False, dev=False): + def matches(self, major=None, minor=None, patch=None, pre=False, dev=False): return ( - self.major == major + (major is None or self.major == major) and (minor is None or self.minor == minor) and (patch is None or self.patch == patch) and (pre is None or self.is_prerelease == pre) @@ -77,7 +101,7 @@ class PythonVersion(object): """ try: - version = parse_version(version) + version = parse_version(str(version)) except TypeError: raise ValueError("Unable to parse version: %s" % version) if not version or not version.release: diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 32e62b18..eff73e22 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -50,31 +50,14 @@ class Finder(object): return self.system_path.which(exe) def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None): - if ( - major - and not minor - and not patch - and not pre - and not dev - and isinstance(major, six.string_types) - ): - from .models import PythonVersion - version_dict = {} - if "." in major: - version_dict = PythonVersion.parse(major) - elif len(major) == 1: - version_dict = { - 'major': int(major), - 'minor': None, - 'patch': None, - 'is_prerelease': False, - 'is_devrelease': False - } + from .models import PythonVersion + if isinstance(major, six.string_types) and pre is None and minor is None and dev is None and patch is None: + version_dict = PythonVersion.parse(major) major = version_dict.get("major", major) minor = version_dict.get("minor", minor) patch = version_dict.get("patch", patch) - pre = version_dict.get("is_prerelease", pre) - dev = version_dict.get("is_devrelease", dev) + pre = version_dict.get("is_prerelease", pre) if pre is not None else pre + dev = version_dict.get("is_devrelease", dev) if dev is not None else dev if os.name == "nt": match = self.windows_finder.find_python_version( major, minor=minor, patch=patch, pre=pre, dev=dev @@ -82,5 +65,5 @@ class Finder(object): if match: return match return self.system_path.find_python_version( - major, minor=minor, patch=patch, pre=pre, dev=dev + major=major, minor=minor, patch=patch, pre=pre, dev=dev ) From b93b287fc9594c174460c37f919507763320e24f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 01:43:13 -0400 Subject: [PATCH 026/101] Try the new code Signed-off-by: Dan Ryan --- pipenv/core.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index c55cc734..de26da9b 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -332,32 +332,34 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - version = [int(v) for v in python.split('.')] - minor = None - patch = None - if len(version) > 1: - minor = version[1] - if len(version) > 2: - patch = version[2] - python_entries = getattr(finder, 'system_path').find_all_python_versions(version[0]) - python_entries = [(entry.py_version.version_tuple, entry) for entry in python_entries] - if minor and patch: - _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor, patch)) - python_entry = _py[1] if _py else None - elif minor: - _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor)) - python_entry = _py[1] if _py else None - else: - _py = first(python_entries) - python_entry = _py[1] if _py else None + # version = [int(v) for v in python.split('.')] + # minor = None + # patch = None + # if len(version) > 1: + # minor = version[1] + # if len(version) > 2: + # patch = version[2] + # python_entries = getattr(finder, 'system_path').find_all_python_versions(version[0]) + # python_entries = [(entry.py_version.version_tuple, entry) for entry in python_entries] + python_entry = finder.find_python_version(python) + # if minor and patch: + # _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor, patch)) + # python_entry = _py[1] if _py else None + # elif minor: + # _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor)) + # python_entry = _py[1] if _py else None + # else: + # _py = first(python_entries) + # python_entry = _py[1] if _py else None if not python_entry: - version_str = "{0}.{1}".format(version[0], minor) if minor else "{0}".format(version[0]) - exe_name = "python{0}".format(version_str) + exe_name = "python{0}".format(python) python_entry = finder.which(exe_name) if python_entry: + click.echo("Found python: {0} => {1}".format(python_entry.as_python.version, python_entry.path.as_posix())) return python_entry.path.as_posix() return None + def ensure_python(three=None, python=None): # Support for the PIPENV_PYTHON environment variable. from .environments import PIPENV_PYTHON From 597cd2d917f7ec6d9a32dc3f5ba0f1da5cf3b7f0 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 01:46:39 -0400 Subject: [PATCH 027/101] Cleanup core and help to use new finder Signed-off-by: Dan Ryan --- pipenv/core.py | 19 ------------------- pipenv/help.py | 3 +-- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index de26da9b..4178b4f2 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -332,30 +332,11 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - # version = [int(v) for v in python.split('.')] - # minor = None - # patch = None - # if len(version) > 1: - # minor = version[1] - # if len(version) > 2: - # patch = version[2] - # python_entries = getattr(finder, 'system_path').find_all_python_versions(version[0]) - # python_entries = [(entry.py_version.version_tuple, entry) for entry in python_entries] python_entry = finder.find_python_version(python) - # if minor and patch: - # _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor, patch)) - # python_entry = _py[1] if _py else None - # elif minor: - # _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor)) - # python_entry = _py[1] if _py else None - # else: - # _py = first(python_entries) - # python_entry = _py[1] if _py else None if not python_entry: exe_name = "python{0}".format(python) python_entry = finder.which(exe_name) if python_entry: - click.echo("Found python: {0} => {1}".format(python_entry.as_python.version, python_entry.path.as_posix())) return python_entry.path.as_posix() return None diff --git a/pipenv/help.py b/pipenv/help.py index 118a2272..93f8f3ff 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -30,8 +30,7 @@ def get_pipenv_diagnostics(): print("Other Python installations in `PATH`:") print("") finder = pythonfinder.Finder(system=False, global_search=True) - python_versions = (getattr(finder, 'system_path').find_all_python_versions(major) for major in (2, 3)) - python_paths = list(chain(*python_versions)) + python_paths = getattr(finder, 'system_path').find_all_python_versions() for python in python_paths: python_version = python.py_version.version python_path = python.path.as_posix() From 0d7afadd63cfaabec1ecd224a7f731ae7bd20543 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 02:17:00 -0400 Subject: [PATCH 028/101] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 2 +- pipenv/vendor/pythonfinder/models/windows.py | 21 ++++++++++++++++---- pipenv/vendor/pythonfinder/pythonfinder.py | 9 +++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 893b6af0..a7255049 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -174,7 +174,7 @@ class SystemPath(object): paths = (self.get_path(k) for k in self.path_order) path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) version_sort = operator.attrgetter("as_python.version_sort") - return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + return (c for c in sorted(path_filter, key=version_sort, reverse=True)) def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): """Search for a specific python version on the path. diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index b6998823..4e5b5b98 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -16,16 +16,29 @@ class WindowsFinder(BaseFinder): version_list = attr.ib(default=attr.Factory(list)) versions = attr.ib() - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): version_matcher = operator.methodcaller( - "matches", major, minor=minor, patch=patch, pre=pre, dev=dev + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) ) - version_sort = operator.attrgetter("version") + version_sort = operator.attrgetter("version_sort") + for c in sorted(py_filter, key=version_sort, reverse=True): + yield c.comes_from + + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): + # version_matcher = operator.methodcaller( + # "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev + # ) + # py_filter = filter( + # None, filter(lambda c: version_matcher(c), self.version_list) + # ) + # version_sort = operator.attrgetter("version_sort") return next( - (c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)), None + self.find_all_python_versions( + major=major, minor=minor, patch=patch, pre=pre, dev=dev + ), None ) @versions.default diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index eff73e22..fa7578d2 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -2,6 +2,7 @@ from __future__ import print_function, absolute_import import os import six +import operator from .models import SystemPath @@ -67,3 +68,11 @@ class Finder(object): return self.system_path.find_python_version( major=major, minor=minor, patch=patch, pre=pre, dev=dev ) + + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): + version_sort = operator.attrgetter("as_python.version_sort") + versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) + if os.name == 'nt': + windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) + versions = list(versions) + list(windows_versions) + return sorted(versions, key=version_sort, reverse=True) From 08cb159e22d8b670daefc09c88feaff8188d8d00 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 02:21:42 -0400 Subject: [PATCH 029/101] Windows fix Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/windows.py | 16 ++++------------ pipenv/vendor/pythonfinder/pythonfinder.py | 5 +++-- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index 4e5b5b98..f33a4807 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -24,21 +24,13 @@ class WindowsFinder(BaseFinder): None, filter(lambda c: version_matcher(c), self.version_list) ) version_sort = operator.attrgetter("version_sort") - for c in sorted(py_filter, key=version_sort, reverse=True): - yield c.comes_from + return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): - # version_matcher = operator.methodcaller( - # "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev - # ) - # py_filter = filter( - # None, filter(lambda c: version_matcher(c), self.version_list) - # ) - # version_sort = operator.attrgetter("version_sort") - return next( - self.find_all_python_versions( + return next(( + v for v in self.find_all_python_versions( major=major, minor=minor, patch=patch, pre=pre, dev=dev - ), None + )), None ) @versions.default diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index fa7578d2..e40405fd 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -71,8 +71,9 @@ class Finder(object): def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): version_sort = operator.attrgetter("as_python.version_sort") - versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) + versions = [] + versions.extend([p for p in self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev)]) if os.name == 'nt': windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) - versions = list(versions) + list(windows_versions) + versions = versions + list(windows_versions) return sorted(versions, key=version_sort, reverse=True) From 2de9505668b40cf3a847bedf6604a7de8bc73c05 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 02:31:22 -0400 Subject: [PATCH 030/101] Fix system path iterator Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index a7255049..893b6af0 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -174,7 +174,7 @@ class SystemPath(object): paths = (self.get_path(k) for k in self.path_order) path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) version_sort = operator.attrgetter("as_python.version_sort") - return (c for c in sorted(path_filter, key=version_sort, reverse=True)) + return [c for c in sorted(path_filter, key=version_sort, reverse=True)] def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): """Search for a specific python version on the path. From 8bb39a412add8db4092484806b9afb43c0aefa56 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 02:41:57 -0400 Subject: [PATCH 031/101] Fix version iteration in pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/pythonfinder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index e40405fd..891a3a17 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -71,8 +71,9 @@ class Finder(object): def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): version_sort = operator.attrgetter("as_python.version_sort") - versions = [] - versions.extend([p for p in self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev)]) + versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) + if not isinstance(versions, list): + versions = [versions,] if os.name == 'nt': windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) versions = versions + list(windows_versions) From 861c075876f40d24b09cff087a205bb59c84962b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 02:42:45 -0400 Subject: [PATCH 032/101] Use new api in help methods Signed-off-by: Dan Ryan --- pipenv/help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/help.py b/pipenv/help.py index 93f8f3ff..ac8d70bd 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -30,7 +30,7 @@ def get_pipenv_diagnostics(): print("Other Python installations in `PATH`:") print("") finder = pythonfinder.Finder(system=False, global_search=True) - python_paths = getattr(finder, 'system_path').find_all_python_versions() + python_paths = finder.find_all_python_versions() for python in python_paths: python_version = python.py_version.version python_path = python.path.as_posix() From 27028e1942969339e9a065dae62cc789dacb2a0f Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 03:22:44 +0800 Subject: [PATCH 033/101] Possibly stray echo --- pipenv/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 4178b4f2..7d7a855b 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -371,7 +371,6 @@ def ensure_python(three=None, python=None): if not python: python = PIPENV_DEFAULT_PYTHON_VERSION path_to_python = find_a_system_python(python) - click.echo("Found path to python: %s" % path_to_python) if not path_to_python and python is not None: # We need to install Python. click.echo( From 9cb3f970bd1eaffb01aecb990a022931294c1573 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 03:53:36 +0800 Subject: [PATCH 034/101] Clean up help module --- pipenv/help.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pipenv/help.py b/pipenv/help.py index ac8d70bd..e8fbf1c8 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -1,12 +1,11 @@ # coding: utf-8 import os +import pprint import sys + import pipenv -from itertools import chain -from pprint import pprint -from .__version__ import __version__ -from .core import project, system_which +from .core import project from .pep508checker import lookup from .vendor import pythonfinder @@ -21,26 +20,25 @@ def print_utf(line): def get_pipenv_diagnostics(): print("
$ pipenv --support") print("") - print("Pipenv version: `{0!r}`".format(__version__)) + print("Pipenv version: `{0!r}`".format(pipenv.__version__)) print("") print("Pipenv location: `{0!r}`".format(os.path.dirname(pipenv.__file__))) print("") print("Python location: `{0!r}`".format(sys.executable)) print("") - print("Other Python installations in `PATH`:") + print("Python installations found:") print("") + finder = pythonfinder.Finder(system=False, global_search=True) python_paths = finder.find_all_python_versions() for python in python_paths: - python_version = python.py_version.version - python_path = python.path.as_posix() - print(" - `{0}`: `{1}`".format(python_version, python_path)) + print(" - `{}`: `{}`".format(python.py_version.version, python.path)) print("") print("PEP 508 Information:") print("") print("```") - pprint(lookup) + pprint.pprint(lookup) print("```") print("") print("System environment variables:") From a2f84421791c12ca9743f4037546da740bd141a7 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 03:53:52 +0800 Subject: [PATCH 035/101] We DONT want sys.executable when searching sys.executable should be the last resort if the user gives us nothing. If we are given ANYTHING, we should avoid using sys.executable. --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 7d7a855b..873d1ff1 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -319,7 +319,7 @@ def find_a_system_python(python): from .vendor.pythonfinder import Finder # system always refers to sys.executable, which could point at a virtualenv # for global searches we most likely want to turn that off - finder = Finder(system=True, global_search=True) + finder = Finder(system=False, global_search=True) python_entry = None if not python: return None From 0b7b0c983a46e3519c4538c6755e64d83d6d3a4f Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 04:17:15 +0800 Subject: [PATCH 036/101] Refactor and docstring for find_a_system_python --- pipenv/core.py | 44 +++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 873d1ff1..dba192bc 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -315,27 +315,37 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): project.write_toml(p) -def find_a_system_python(python): - from .vendor.pythonfinder import Finder - # system always refers to sys.executable, which could point at a virtualenv - # for global searches we most likely want to turn that off - finder = Finder(system=False, global_search=True) - python_entry = None - if not python: +def find_a_system_python(line): + """Find a Python installation from a given line. + + This tries to parse the line in various of ways: + + * Looks like an absolute path? Use it directly. + * Looks like a py.exe call? Use py.exe to get the executable. + * Starts with "py" something? Looks like a python command. Try to find it + in PATH, and use it directly. + * Search for "python" and "pythonX.Y" executables in PATH to find a match. + * Nothing fits, return None. + """ + if not line: return None - # when using the python launcher on windows we can find the versions ourselves - if os.name == 'nt' and python.startswith("py -"): - python = python[len("py -"):] - if python.startswith("py"): - python_entry = finder.which(python) + if os.path.isabs(line): + return line + from .vendor.pythonfinder import Finder + finder = Finder(system=False, global_search=True) + if ((line.startswith("py ") or line.startswith("py.exe ")) + and finder.which("py.exe")): + import subprocess + return subprocess.check_output( + '{} -c "import sys; print(sys.executable)"'.format(line), + ) + if line.startswith("py"): + python_entry = finder.which(line) if python_entry: return python_entry.path.as_posix() - if os.path.isabs(python): - return python - python_entry = finder.find_python_version(python) + python_entry = finder.find_python_version(line) if not python_entry: - exe_name = "python{0}".format(python) - python_entry = finder.which(exe_name) + python_entry = finder.which("python{0}".format(line)) if python_entry: return python_entry.path.as_posix() return None From 07a64777b4f0394392cce425b304ece59b27228e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 04:19:23 +0800 Subject: [PATCH 037/101] Unused --- pipenv/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index dba192bc..710483cb 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -12,7 +12,6 @@ import click_completion import crayons import dotenv import delegator -from first import first import pipfile from blindspin import spinner import six From 3a734f1ef0de69b2a4b7d83d6ee9c664470271d0 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 04:25:16 +0800 Subject: [PATCH 038/101] Proper subprocess decoding --- pipenv/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 710483cb..0cabc05b 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -335,9 +335,12 @@ def find_a_system_python(line): if ((line.startswith("py ") or line.startswith("py.exe ")) and finder.which("py.exe")): import subprocess - return subprocess.check_output( + output = subprocess.check_output( '{} -c "import sys; print(sys.executable)"'.format(line), ) + if not isinstance(output, str): + output = output.decode(sys.getdefaultencoding()) + return output.strip() if line.startswith("py"): python_entry = finder.which(line) if python_entry: From 10749cc66cb531101cc1e0e434a86fbd0f0dbb8e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 04:34:18 +0800 Subject: [PATCH 039/101] Don't fallback if a command is not found --- pipenv/core.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 0cabc05b..b5301c01 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -345,6 +345,7 @@ def find_a_system_python(line): python_entry = finder.which(line) if python_entry: return python_entry.path.as_posix() + return None python_entry = finder.find_python_version(line) if not python_entry: python_entry = finder.which("python{0}".format(line)) @@ -849,21 +850,16 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): # Actually create the virtualenv. with spinner(): - try: - c = delegator.run(cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config) - except OSError: - click.echo( - "{0}: it looks like {1} is not in your {2}. " - "We cannot continue until this is resolved." - "".format( - crayons.red("Warning", bold=True), - crayons.red(cmd[0]), - crayons.normal("PATH", bold=True), - ), - err=True, - ) - sys.exit(1) + c = delegator.run( + cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config, + ) click.echo(crayons.blue(c.out), err=True) + if c.return_code != 0: + click.echo(crayons.blue(c.err), err=True) + click.echo("{0}: Failed to create virtual environment.".format( + crayons.red("Warning", bold=True), + ), err=True) + sys.exit(1) # Associate project directory with the environment. # This mimics Pew's "setproject". From a709bf845f4d0422d6dd0b682aca4c702d039680 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 02:37:32 -0400 Subject: [PATCH 040/101] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/__init__.py | 41 ++++-- pipenv/vendor/pythonfinder/models/path.py | 130 +++++++++++++----- pipenv/vendor/pythonfinder/models/pyenv.py | 16 +++ pipenv/vendor/pythonfinder/models/python.py | 29 +++- pipenv/vendor/pythonfinder/models/windows.py | 19 ++- pipenv/vendor/pythonfinder/pythonfinder.py | 32 +++-- pipenv/vendor/pythonfinder/utils.py | 13 +- 7 files changed, 217 insertions(+), 63 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index 0c6f0134..b1b56d43 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -3,6 +3,7 @@ from __future__ import print_function, absolute_import import abc import operator import six +from itertools import chain from ..utils import KNOWN_EXTS @@ -42,28 +43,52 @@ class BasePath(object): found = next((children[(self.path / child).as_posix()] for child in valid_names if (self.path / child).as_posix() in children), None) return found - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + """Search for a specific python version on the path. Return all copies + + :param major: Major python version to search for. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. + :rtype: List[:class:`~pythonfinder.models.PathEntry`] + """ + + sub_finder = operator.methodcaller( + "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + ) + if not self.is_dir: + return sub_finder(self) + path_filter = filter(None, (sub_finder(p) for p in self.children.values())) + version_sort = operator.attrgetter("as_python.version_sort") + return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): """Search or self for the specified Python version and return the first match. :param major: Major version number. :type major: int - :param minor: Minor python version, defaults to None - :param minor: int, optional - :param patch: Patch python version, defaults to None - :param patch: int, optional + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") if not self.is_dir: - if self.is_python and self.as_python.matches(major=major, minor=minor, patch=patch, pre=pre, dev=dev): + if self.is_python and version_matcher(self.as_python): return self return - finder = ((child, child.as_python) for child in self.children.values() if child.is_python and child.as_python) + finder = ((child, child.as_python) for child in chain(*filter(None, self.pythons.values())) if child.as_python) py_filter = filter( None, filter(lambda child: version_matcher(child[1]), finder) ) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 893b6af0..5e1807ab 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -1,10 +1,12 @@ # -*- coding=utf-8 -*- from __future__ import print_function, absolute_import import attr +import copy import operator import os import sys from collections import defaultdict +from itertools import chain from . import BasePath from .python import PythonVersion from ..environment import PYENV_INSTALLED, PYENV_ROOT @@ -13,7 +15,7 @@ from ..utils import ( optional_instance_of, filter_pythons, path_is_known_executable, - is_python_name, + looks_like_python, ensure_path, fs_str ) @@ -31,32 +33,49 @@ class SystemPath(object): _executables = attr.ib(default=attr.Factory(list)) _python_executables = attr.ib(default=attr.Factory(list)) path_order = attr.ib(default=attr.Factory(list)) - python_version_dict = attr.ib() + python_version_dict = attr.ib(default=attr.Factory(defaultdict)) only_python = attr.ib(default=False) pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) system = attr.ib(default=False) + __finders = attr.ib(default=attr.Factory(list)) + + def _register_finder(self, finder): + if not finder in self.__finders: + self.__finders.append(finder) + @property def executables(self): if not self._executables: - self._executables = [p for p in self.paths.values() if p.is_executable] + self._executables = [p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable] return self._executables @property def python_executables(self): + python_executables = {} if not self._python_executables: - self._python_executables = [p for p in self.paths.values() if p.is_python] + for child in self.paths.values(): + if child.pythons: + python_executables.update(dict(child.pythons)) + for finder in self.__finders: + if finder.pythons: + python_executables.update(dict(finder.pythons)) + self._python_executables = python_executables return self._python_executables - @python_version_dict.default def get_python_version_dict(self): version_dict = defaultdict(list) - for p in self.python_executables: - try: - version_object = PythonVersion.from_path(p) - except (ValueError, InvalidPythonVersion): + for finder in self.__finders: + for version, entry in finder.versions.items(): + if entry not in version_dict[version]: + version_dict[version].append(entry) + for p, entry in self.python_executables.items(): + version = entry.as_python + if not version: continue - version_dict[version_object.version_tuple].append(version_object) + version = version.version_tuple + if version and entry not in version_dict[version]: + version_dict[version].append(entry) return version_dict def __attrs_post_init__(self): @@ -73,7 +92,7 @@ class SystemPath(object): else: bin_dir = 'bin' if venv and (self.system or self.global_search): - p = Path(venv) + p = ensure_path(venv) self.path_order = [(p / bin_dir).as_posix()] + self.path_order self.paths[p] = PathEntry.create( path=p, is_root=True, only_python=False @@ -84,9 +103,10 @@ class SystemPath(object): if syspath_bin.name != bin_dir and syspath_bin.joinpath(bin_dir).exists(): syspath_bin = syspath_bin / bin_dir self.path_order = [syspath_bin.as_posix()] + self.path_order - self.paths[syspath_bin.as_posix()] = PathEntry.create( + self.paths[syspath_bin] = PathEntry.create( path=syspath_bin, is_root=True, only_python=False ) + self.python_version_dict = self.get_python_version_dict() def _setup_pyenv(self): from .pyenv import PyenvFinder @@ -110,6 +130,7 @@ class SystemPath(object): before_path + [p.path.as_posix() for p in root_paths] + after_path ) self.paths.update({p.path: p for p in root_paths}) + self._register_finder(self.pyenv_finder) def _setup_windows(self): from .windows import WindowsFinder @@ -119,15 +140,17 @@ class SystemPath(object): path_addition = [p.path.as_posix() for p in root_paths] self.path_order = self.path_order[:] + path_addition self.paths.update({p.path: p for p in root_paths}) + self._register_finder(self.windows_finder) def get_path(self, path): - path = Path(path) + path = ensure_path(path) _path = self.paths.get(path.as_posix()) if not _path and path.as_posix() in self.path_order: - self.paths[path.as_posix()] = PathEntry.create( - path=path.resolve(), is_root=True, only_python=self.only_python + _path = PathEntry.create( + path=path.absolute(), is_root=True, only_python=self.only_python ) - return self.paths.get(path.as_posix()) + self.paths[path.as_posix()] = _path + return _path def find_all(self, executable): """Search the path for an executable. Return all copies. @@ -151,21 +174,22 @@ class SystemPath(object): filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) return next((f for f in filtered), None) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. :type major: int - :param minor: Minor python version to search for, defaults to None - :param minor: int, optional - :param path: Patch python version to search for, defaults to None - :param path: int, optional + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev + "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) @@ -176,22 +200,33 @@ class SystemPath(object): version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): """Search for a specific python version on the path. :param major: Major python version to search for. :type major: int - :param minor: Minor python version to search for, defaults to None - :param minor: int, optional - :param path: Patch python version to search for, defaults to None - :param path: int, optional + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None :return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. :rtype: :class:`~pythonfinder.models.PathEntry` """ sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev + "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) + if major and minor and patch: + _tuple_pre = pre if pre is not None else False + _tuple_dev = dev if dev is not None else False + version_tuple = (major, minor_, patch, _tuple_pre, _tuple_dev) + version_tuple_pre = (major, minor, patch, True, False) + version = self.python_version_dict.get(version_tuple) + if not version: + version = self.python_version_dict.get(version_tuple_pre) + if version: + return first(version.comes_from) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: @@ -243,7 +278,7 @@ class PathEntry(BasePath): is_root = attr.ib(default=True) only_python = attr.ib(default=False) py_version = attr.ib(default=None) - pythons = attr.ib(default=None) + pythons = attr.ib() def __str__(self): return fs_str('{0}'.format(self.path.as_posix())) @@ -259,11 +294,27 @@ class PathEntry(BasePath): def children(self): if not self._children and self.is_dir and self.is_root: self._children = { - child.as_posix(): PathEntry(path=child, is_root=False) + child.as_posix(): PathEntry.create(path=child, is_root=False) for child in self._filter_children() } + elif not self.is_dir: + return {self.path.as_posix(): self} return self._children + @pythons.default + def get_pythons(self): + pythons = defaultdict() + if self.is_dir: + for path, entry in self.children.items(): + _path = ensure_path(entry.path) + if entry.is_python: + pythons[_path.as_posix()] = entry + else: + if self.is_python: + _path = ensure_path(self.path) + pythons[_path.as_posix()] = copy.deepcopy(self) + return pythons + @property def as_python(self): if not self.is_dir and self.is_python: @@ -292,9 +343,14 @@ class PathEntry(BasePath): """ target = ensure_path(path) - _new = cls( - path=target, is_root=is_root, only_python=only_python, pythons=pythons - ) + creation_args = { + "path": target, + "is_root": is_root, + "only_python": only_python + } + if pythons: + creation_args["pythons"] = pythons + _new = cls(**creation_args) if pythons and only_python: children = {} for pth, python in pythons.items(): @@ -311,7 +367,11 @@ class PathEntry(BasePath): @property def is_dir(self): - return self.path.is_dir() + try: + ret_val = self.path.is_dir() + except OSError: + ret_val = False + return ret_val @property def is_executable(self): @@ -320,7 +380,7 @@ class PathEntry(BasePath): @property def is_python(self): return self.is_executable and ( - self.py_version or is_python_name(self.path.name) + self.py_version or looks_like_python(self.path.name) ) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 8545ac59..e61e7153 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -18,6 +18,7 @@ except ImportError: class PyenvFinder(BaseFinder): root = attr.ib(default=None, validator=optional_instance_of(Path)) versions = attr.ib() + pythons = attr.ib() @versions.default def get_versions(self): @@ -34,6 +35,21 @@ class PyenvFinder(BaseFinder): versions[version_tuple] = VersionPath.create(path=p.resolve(), only_python=True) return versions + @pythons.default + def get_pythons(self): + pythons = defaultdict() + for v in self.versions.values(): + for p in v.paths.values(): + _path = p.path + try: + _path = _path.resolve() + except OSError: + _path = _path.absolute() + _path = _path.as_posix() + if p.is_python: + pythons[_path] = p + return pythons + @classmethod def create(cls, root): root = ensure_path(root) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 4ff364fa..687f1d43 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -2,6 +2,7 @@ from __future__ import print_function, absolute_import import attr import copy +from collections import defaultdict import platform from packaging.version import parse as parse_version, Version from ..environment import SYSTEM_ARCH @@ -67,13 +68,14 @@ class PythonVersion(object): self.is_devrelease, ) - def matches(self, major=None, minor=None, patch=None, pre=False, dev=False): + def matches(self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None): return ( (major is None or self.major == major) and (minor is None or self.minor == minor) and (patch is None or self.patch == patch) and (pre is None or self.is_prerelease == pre) and (dev is None or self.is_devrelease == dev) + and (arch is None or self.architecture == arch) ) def as_major(self): @@ -143,7 +145,7 @@ class PythonVersion(object): from .path import PathEntry if not isinstance(path, PathEntry): - path = PathEntry(path) + path = PathEntry.create(path, is_root=False, only_python=True) if not path.is_python: raise ValueError("Not a valid python path: %s" % path.path) return @@ -192,3 +194,26 @@ class PythonVersion(object): @classmethod def create(cls, **kwargs): return cls(**kwargs) + + +@attr.s +class VersionMap(object): + versions = attr.ib(default=attr.Factory(defaultdict(list))) + + def add_entry(self, entry): + version = entry.as_python + if version: + entries = versions[version.version_tuple] + paths = {p.path for p in self.versions.get(version.version_tuple, [])} + if entry.path not in paths: + self.versions[version.version_tuple].append(entry) + + def merge(self, target): + for version, entries in target.versions.items(): + if version not in self.versions: + self.versions[version] = entries + else: + current_entries = {p.path for p in self.versions.get(version)} + new_entries = {p.path for p in entries} + new_entries -= current_entries + self.versions[version].append([e for e in entries if e.path in new_entries]) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index f33a4807..90f2d803 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -5,7 +5,7 @@ import operator from collections import defaultdict from . import BaseFinder from .path import PathEntry -from .python import PythonVersion +from .python import PythonVersion, VersionMap from ..exceptions import InvalidPythonVersion from ..utils import ensure_path @@ -15,10 +15,11 @@ class WindowsFinder(BaseFinder): paths = attr.ib(default=attr.Factory(list)) version_list = attr.ib(default=attr.Factory(list)) versions = attr.ib() + pythons = attr.ib() - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -26,10 +27,10 @@ class WindowsFinder(BaseFinder): version_sort = operator.attrgetter("version_sort") return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): return next(( v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None )), None ) @@ -57,6 +58,14 @@ class WindowsFinder(BaseFinder): self.paths.append(base_dir) return versions + @pythons.default + def get_pythons(self): + pythons = defaultdict() + for version in self.version_list: + _path = ensure_path(version.comes_from.path) + pythons[_path.as_posix()] = version.comes_from + return pythons + @classmethod def create(cls): return cls() diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 891a3a17..c74eadeb 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -50,15 +50,20 @@ class Finder(object): def which(self, exe): return self.system_path.which(exe) - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, arch=None): from .models import PythonVersion if isinstance(major, six.string_types) and pre is None and minor is None and dev is None and patch is None: + if arch is None and '-' in major: + major, arch = major.rsplit('-', 1) + if not arch.isnumeric(): + major = "{0}-{1}".format(major, arch) version_dict = PythonVersion.parse(major) major = version_dict.get("major", major) minor = version_dict.get("minor", minor) patch = version_dict.get("patch", patch) - pre = version_dict.get("is_prerelease", pre) if pre is not None else pre - dev = version_dict.get("is_devrelease", dev) if dev is not None else dev + pre = version_dict.get("is_prerelease", pre) if pre is None else pre + dev = version_dict.get("is_devrelease", dev) if dev is None else dev + arch = version_dict.get("architecture", arch) if arch is None else arch if os.name == "nt": match = self.windows_finder.find_python_version( major, minor=minor, patch=patch, pre=pre, dev=dev @@ -66,15 +71,24 @@ class Finder(object): if match: return match return self.system_path.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): version_sort = operator.attrgetter("as_python.version_sort") - versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) + versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) if not isinstance(versions, list): versions = [versions,] if os.name == 'nt': - windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) - versions = versions + list(windows_versions) - return sorted(versions, key=version_sort, reverse=True) + windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) + versions = list(windows_versions) + versions + paths = sorted(versions, key=version_sort, reverse=True) + path_map = {} + for path in paths: + try: + resolved_path = path.path.resolve() + except OSError: + resolved_path = path.path.absolute() + if not path_map.get(resolved_path.as_posix()): + path_map[resolved_path.as_posix()] = path + return list(path_map.values()) diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 0cc5370a..df837ea9 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -72,7 +72,7 @@ def path_is_known_executable(path): ) -def is_python_name(name): +def looks_like_python(name): rules = ["*python", "*python?", "*python?.?", "*python?.?m"] match_rules = [] for rule in rules: @@ -88,7 +88,7 @@ def is_python_name(name): def path_is_python(path): - return path_is_executable(path) and is_python_name(path.name) + return path_is_executable(path) and looks_like_python(path.name) def ensure_path(path): @@ -101,8 +101,13 @@ def ensure_path(path): """ if isinstance(path, Path): - return Path(os.path.expandvars(path.as_posix())) - return Path(os.path.expandvars(path)) + path = path.as_posix() + path = Path(os.path.expandvars(path)) + try: + path = path.resolve() + except OSError: + path = path.absolute() + return path def _filter_none(k, v): From 6518325a3c5be27a8d4661ac9084ef921c2d8759 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 02:37:55 -0400 Subject: [PATCH 041/101] Stop using subprocess to interface with windows finder Signed-off-by: Dan Ryan --- pipenv/core.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index b5301c01..2fc60b45 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -333,14 +333,8 @@ def find_a_system_python(line): from .vendor.pythonfinder import Finder finder = Finder(system=False, global_search=True) if ((line.startswith("py ") or line.startswith("py.exe ")) - and finder.which("py.exe")): - import subprocess - output = subprocess.check_output( - '{} -c "import sys; print(sys.executable)"'.format(line), - ) - if not isinstance(output, str): - output = output.decode(sys.getdefaultencoding()) - return output.strip() + and os.name == 'nt'): + line = line.split(" ", 1)[1] if line.startswith("py"): python_entry = finder.which(line) if python_entry: From febfc6e907fe24cf20ba7cefc00e83a64079a8f2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 02:45:57 -0400 Subject: [PATCH 042/101] Better algorithm for find_all_versions Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/pythonfinder.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index c74eadeb..792f5991 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -76,6 +76,11 @@ class Finder(object): def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): version_sort = operator.attrgetter("as_python.version_sort") + python_version_dict = getattr(self.system_path, 'python_version_dict') + if python_version_dict: + paths = [path for version in python_version_dict.values() for path in version] + paths = sorted(paths, key=version_sort, reverse=True) + return paths versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) if not isinstance(versions, list): versions = [versions,] From cc9c5a3ff8fecd6dbcae466af38a4f288fb78f53 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 02:58:37 -0400 Subject: [PATCH 043/101] Strip dashes from `py -n` commands Signed-off-by: Dan Ryan --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 2fc60b45..f32230bc 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -334,7 +334,7 @@ def find_a_system_python(line): finder = Finder(system=False, global_search=True) if ((line.startswith("py ") or line.startswith("py.exe ")) and os.name == 'nt'): - line = line.split(" ", 1)[1] + line = line.split(" ", 1)[1].lstrip("-") if line.startswith("py"): python_entry = finder.which(line) if python_entry: From 95f0df79e8a51d9fa3d4b956420dce70b64100d9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 03:25:18 -0400 Subject: [PATCH 044/101] Fix pythonfinder bug unnesting python versions Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/__init__.py | 4 ++-- pipenv/vendor/pythonfinder/utils.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index b1b56d43..4d906544 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -4,7 +4,7 @@ import abc import operator import six from itertools import chain -from ..utils import KNOWN_EXTS +from ..utils import KNOWN_EXTS, unnest @six.add_metaclass(abc.ABCMeta) @@ -88,7 +88,7 @@ class BasePath(object): if self.is_python and version_matcher(self.as_python): return self return - finder = ((child, child.as_python) for child in chain(*filter(None, self.pythons.values())) if child.as_python) + finder = ((child, child.as_python) for child in unnest(self.pythons.values()) if child.as_python) py_filter = filter( None, filter(lambda child: version_matcher(child[1]), finder) ) diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index df837ea9..fdc54381 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -8,6 +8,7 @@ import subprocess import sys from fnmatch import fnmatch from .exceptions import InvalidPythonVersion +from itertools import chain try: from pathlib import Path @@ -137,3 +138,9 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() + + +def unnest(item): + if isinstance(next((i for i in item), None), (list, tuple)): + return chain(*filter(None, item)) + return chain(filter(None, item)) From b5bd420c05e51e891d07d40d2069b234c0273740 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 03:34:19 -0400 Subject: [PATCH 045/101] Fix click encoding for terminal outputs Signed-off-by: Dan Ryan --- pipenv/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index f32230bc..4718ab99 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -847,10 +847,10 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): c = delegator.run( cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config, ) - click.echo(crayons.blue(c.out), err=True) + click.echo(u"{0}".format(crayons.blue(c.out), err=True)) if c.return_code != 0: - click.echo(crayons.blue(c.err), err=True) - click.echo("{0}: Failed to create virtual environment.".format( + click.echo(u"{0}".format(crayons.blue(c.err), err=True)) + click.echo(u"{0}: Failed to create virtual environment.".format( crayons.red("Warning", bold=True), ), err=True) sys.exit(1) From 9e10e72493ee96a8ab53eea250881e77c03862a8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 03:48:31 -0400 Subject: [PATCH 046/101] Block before getting outputs Signed-off-by: Dan Ryan --- pipenv/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 4718ab99..88b4c931 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -847,9 +847,10 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): c = delegator.run( cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config, ) - click.echo(u"{0}".format(crayons.blue(c.out), err=True)) + c.block() + click.echo(crayons.blue({0}.format(c.out)), err=True) if c.return_code != 0: - click.echo(u"{0}".format(crayons.blue(c.err), err=True)) + click.echo(crayons.blue("{0}".format(c.err)), err=True) click.echo(u"{0}: Failed to create virtual environment.".format( crayons.red("Warning", bold=True), ), err=True) From 2f85e9bcc753143db22dfe8034cb6ecd20a146b8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 03:54:01 -0400 Subject: [PATCH 047/101] Add quotes Signed-off-by: Dan Ryan --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 88b4c931..7992797f 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -848,7 +848,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config, ) c.block() - click.echo(crayons.blue({0}.format(c.out)), err=True) + click.echo(crayons.blue("{0}".format(c.out)), err=True) if c.return_code != 0: click.echo(crayons.blue("{0}".format(c.err)), err=True) click.echo(u"{0}: Failed to create virtual environment.".format( From 4b26aa8c9f864e55a17dd8f57de6b6f1b34de0f1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 04:30:42 -0400 Subject: [PATCH 048/101] Fix aggregation for support Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 5 ++++- pipenv/vendor/pythonfinder/models/pyenv.py | 7 +------ pipenv/vendor/pythonfinder/pythonfinder.py | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 5e1807ab..11548f23 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -67,7 +67,10 @@ class SystemPath(object): version_dict = defaultdict(list) for finder in self.__finders: for version, entry in finder.versions.items(): - if entry not in version_dict[version]: + if isinstance(entry, VersionPath): + paths = [p for p in entry.paths.values() if p.is_python and p not in version_dict[version]] + version_dict[version].extend(paths) + elif entry not in version_dict[version]: version_dict[version].append(entry) for p, entry in self.python_executables.items(): version = entry.as_python diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index e61e7153..13476b6b 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -40,12 +40,7 @@ class PyenvFinder(BaseFinder): pythons = defaultdict() for v in self.versions.values(): for p in v.paths.values(): - _path = p.path - try: - _path = _path.resolve() - except OSError: - _path = _path.absolute() - _path = _path.as_posix() + _path = ensure_path(p.path) if p.is_python: pythons[_path] = p return pythons diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 792f5991..2cc82d86 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -78,7 +78,7 @@ class Finder(object): version_sort = operator.attrgetter("as_python.version_sort") python_version_dict = getattr(self.system_path, 'python_version_dict') if python_version_dict: - paths = [path for version in python_version_dict.values() for path in version] + paths = filter(None, [path for version in python_version_dict.values() for path in version]) paths = sorted(paths, key=version_sort, reverse=True) return paths versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) From 216f639ae25fe8e56ce138e079b2b2ace5fff2fd Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 22:13:24 +0800 Subject: [PATCH 049/101] Don't process the py line twice --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 7992797f..c3bf6612 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -335,7 +335,7 @@ def find_a_system_python(line): if ((line.startswith("py ") or line.startswith("py.exe ")) and os.name == 'nt'): line = line.split(" ", 1)[1].lstrip("-") - if line.startswith("py"): + elif line.startswith("py"): python_entry = finder.which(line) if python_entry: return python_entry.path.as_posix() From 6e4205a1348e0c53a1b4598074653be9f696cfd7 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 22:20:48 +0800 Subject: [PATCH 050/101] Pass arch into Windows finder This is still not working though. --- pipenv/vendor/pythonfinder/pythonfinder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 2cc82d86..75925075 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -66,7 +66,7 @@ class Finder(object): arch = version_dict.get("architecture", arch) if arch is None else arch if os.name == "nt": match = self.windows_finder.find_python_version( - major, minor=minor, patch=patch, pre=pre, dev=dev + major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, ) if match: return match From bf8d0b9592a4a27f958c789abcd250b1fdba1d50 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 18:23:55 -0400 Subject: [PATCH 051/101] Update pythonfinder for better windows support Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/python.py | 7 ++++++- pipenv/vendor/pythonfinder/pythonfinder.py | 6 ++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 687f1d43..1bc34cc0 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -69,6 +69,8 @@ class PythonVersion(object): ) def matches(self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None): + if arch and arch.isnumeric(): + arch = '{0}bit'.format(arch) return ( (major is None or self.major == major) and (minor is None or self.minor == minor) @@ -180,7 +182,7 @@ class PythonVersion(object): creation_dict.update( { "architecture": getattr( - launcher_entry, "sys_architecture", SYSTEM_ARCH + launcher_entry.info, "sys_architecture", SYSTEM_ARCH ), "executable": exe_path, } @@ -193,6 +195,9 @@ class PythonVersion(object): @classmethod def create(cls, **kwargs): + if 'architecture' in kwargs: + if kwargs['architecture'].isnumeric(): + kwargs['architecture'] = '{0}bit'.format(kwargs['architecture']) return cls(**kwargs) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 75925075..8e775ffd 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -57,6 +57,8 @@ class Finder(object): major, arch = major.rsplit('-', 1) if not arch.isnumeric(): major = "{0}-{1}".format(major, arch) + else: + arch = "{0}bit".format(arch) version_dict = PythonVersion.parse(major) major = version_dict.get("major", major) minor = version_dict.get("minor", minor) @@ -66,7 +68,7 @@ class Finder(object): arch = version_dict.get("architecture", arch) if arch is None else arch if os.name == "nt": match = self.windows_finder.find_python_version( - major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, + major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) if match: return match @@ -78,7 +80,7 @@ class Finder(object): version_sort = operator.attrgetter("as_python.version_sort") python_version_dict = getattr(self.system_path, 'python_version_dict') if python_version_dict: - paths = filter(None, [path for version in python_version_dict.values() for path in version]) + paths = filter(None, [path for version in python_version_dict.values() for path in version if path.as_python]) paths = sorted(paths, key=version_sort, reverse=True) return paths versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) From 21cbf0c5b8806d6fc7c02e329b6f8788b4b34ddf Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 20:12:41 -0400 Subject: [PATCH 052/101] Update pythonfinder to default patch versions to 0 Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/python.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 1bc34cc0..4c6aa1f6 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -19,7 +19,7 @@ except ImportError: class PythonVersion(object): major = attr.ib(default=0) minor = attr.ib(default=None) - patch = attr.ib(default=None) + patch = attr.ib(default=0) is_prerelease = attr.ib(default=False) is_postrelease = attr.ib(default=False) is_devrelease = attr.ib(default=False) @@ -47,7 +47,7 @@ class PythonVersion(object): return ( self.major, self.minor, - self.patch, + self.patch if self.patch else 0, release_sort ) From b7daa2525b49e2d8c64dddc1b3a68d436677cb06 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 20:27:42 -0400 Subject: [PATCH 053/101] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 29 ++++++++++++++-------- pipenv/vendor/pythonfinder/pythonfinder.py | 6 ++--- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 11548f23..9315ae23 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -38,11 +38,11 @@ class SystemPath(object): pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) system = attr.ib(default=False) - __finders = attr.ib(default=attr.Factory(list)) + __finders = attr.ib(default=attr.Factory(dict)) - def _register_finder(self, finder): - if not finder in self.__finders: - self.__finders.append(finder) + def _register_finder(self, finder_name, finder): + if finder_name not in self.__finders: + self.__finders[finder_name] = finder @property def executables(self): @@ -57,7 +57,7 @@ class SystemPath(object): for child in self.paths.values(): if child.pythons: python_executables.update(dict(child.pythons)) - for finder in self.__finders: + for finder_name, finder in self.__finders.items(): if finder.pythons: python_executables.update(dict(finder.pythons)) self._python_executables = python_executables @@ -65,12 +65,19 @@ class SystemPath(object): def get_python_version_dict(self): version_dict = defaultdict(list) - for finder in self.__finders: + for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): + if finder_name == 'windows': + if entry not in version_dict[version]: + version_dict[version].append(entry) + continue if isinstance(entry, VersionPath): - paths = [p for p in entry.paths.values() if p.is_python and p not in version_dict[version]] - version_dict[version].extend(paths) - elif entry not in version_dict[version]: + for path in entry.paths.values(): + if path not in version_dict[version] and path.is_python: + version_dict[version].append(path) + continue + continue + elif entry not in version_dict[version] and entry.is_python: version_dict[version].append(entry) for p, entry in self.python_executables.items(): version = entry.as_python @@ -133,7 +140,7 @@ class SystemPath(object): before_path + [p.path.as_posix() for p in root_paths] + after_path ) self.paths.update({p.path: p for p in root_paths}) - self._register_finder(self.pyenv_finder) + self._register_finder('pyenv', self.pyenv_finder) def _setup_windows(self): from .windows import WindowsFinder @@ -143,7 +150,7 @@ class SystemPath(object): path_addition = [p.path.as_posix() for p in root_paths] self.path_order = self.path_order[:] + path_addition self.paths.update({p.path: p for p in root_paths}) - self._register_finder(self.windows_finder) + self._register_finder('windows', self.windows_finder) def get_path(self, path): path = ensure_path(path) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 8e775ffd..fa8706f1 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -86,9 +86,9 @@ class Finder(object): versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) if not isinstance(versions, list): versions = [versions,] - if os.name == 'nt': - windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) - versions = list(windows_versions) + versions + # if os.name == 'nt': + # windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) + # versions = list(windows_versions) + versions paths = sorted(versions, key=version_sort, reverse=True) path_map = {} for path in paths: From 1b69089cb16318fdbef048278578b997efc71cfa Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 23:13:13 -0400 Subject: [PATCH 054/101] Cached properties! Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/__init__.py | 5 +- pipenv/vendor/pythonfinder/models/path.py | 84 ++++++++++--------- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index 4d906544..7cf0fadf 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -57,8 +57,9 @@ class BasePath(object): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ + call_method = "find_all_python_versions" if self.is_dir else "find_python_version" sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + call_method, major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) if not self.is_dir: return sub_finder(self) @@ -85,7 +86,7 @@ class BasePath(object): is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") if not self.is_dir: - if self.is_python and version_matcher(self.as_python): + if self.is_python and self.as_python and version_matcher(self.as_python): return self return finder = ((child, child.as_python) for child in unnest(self.pythons.values()) if child.as_python) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 9315ae23..6857d17d 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -6,6 +6,7 @@ import operator import os import sys from collections import defaultdict +from cached_property import cached_property from itertools import chain from . import BasePath from .python import PythonVersion @@ -17,7 +18,8 @@ from ..utils import ( path_is_known_executable, looks_like_python, ensure_path, - fs_str + fs_str, + unnest, ) try: @@ -37,6 +39,7 @@ class SystemPath(object): only_python = attr.ib(default=False) pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) system = attr.ib(default=False) + _version_dict = attr.ib(default=attr.Factory(defaultdict)) __finders = attr.ib(default=attr.Factory(dict)) @@ -44,49 +47,48 @@ class SystemPath(object): if finder_name not in self.__finders: self.__finders[finder_name] = finder - @property + @cached_property def executables(self): - if not self._executables: - self._executables = [p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable] - return self._executables + self.executables = [p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable] + return self.executables - @property + @cached_property def python_executables(self): python_executables = {} - if not self._python_executables: - for child in self.paths.values(): - if child.pythons: - python_executables.update(dict(child.pythons)) - for finder_name, finder in self.__finders.items(): - if finder.pythons: - python_executables.update(dict(finder.pythons)) - self._python_executables = python_executables + for child in self.paths.values(): + if child.pythons: + python_executables.update(dict(child.pythons)) + for finder_name, finder in self.__finders.items(): + if finder.pythons: + python_executables.update(dict(finder.pythons)) + self._python_executables = python_executables return self._python_executables - def get_python_version_dict(self): - version_dict = defaultdict(list) + @cached_property + def version_dict(self): + self._version_dict = defaultdict(list) for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): if finder_name == 'windows': - if entry not in version_dict[version]: - version_dict[version].append(entry) + if entry not in self._version_dict[version]: + self._version_dict[version].append(entry) continue if isinstance(entry, VersionPath): for path in entry.paths.values(): - if path not in version_dict[version] and path.is_python: - version_dict[version].append(path) + if path not in self._version_dict[version] and path.is_python: + self._version_dict[version].append(path) continue continue - elif entry not in version_dict[version] and entry.is_python: - version_dict[version].append(entry) + elif entry not in self._version_dict[version] and entry.is_python: + self._version_dict[version].append(entry) for p, entry in self.python_executables.items(): version = entry.as_python if not version: continue version = version.version_tuple - if version and entry not in version_dict[version]: - version_dict[version].append(entry) - return version_dict + if version and entry not in self._version_dict[version]: + self._version_dict[version].append(entry) + return self._version_dict def __attrs_post_init__(self): #: slice in pyenv @@ -116,7 +118,6 @@ class SystemPath(object): self.paths[syspath_bin] = PathEntry.create( path=syspath_bin, is_root=True, only_python=False ) - self.python_version_dict = self.get_python_version_dict() def _setup_pyenv(self): from .pyenv import PyenvFinder @@ -199,14 +200,14 @@ class SystemPath(object): """ sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "find_all_python_versions", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version paths = (self.get_path(k) for k in self.path_order) - path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) + path_filter = filter(None, unnest((sub_finder(p) for p in paths if p is not None))) version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] @@ -232,11 +233,6 @@ class SystemPath(object): _tuple_dev = dev if dev is not None else False version_tuple = (major, minor_, patch, _tuple_pre, _tuple_dev) version_tuple_pre = (major, minor, patch, True, False) - version = self.python_version_dict.get(version_tuple) - if not version: - version = self.python_version_dict.get(version_tuple_pre) - if version: - return first(version.comes_from) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: @@ -244,9 +240,15 @@ class SystemPath(object): paths = (self.get_path(k) for k in self.path_order) path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) version_sort = operator.attrgetter("as_python.version_sort") - return next( + ver = next( (c for c in sorted(path_filter, key=version_sort, reverse=True)), None ) + if ver: + if ver.as_python.version_tuple[:5] in self.python_version_dict: + self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) + else: + self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver,] + return ver @classmethod def create(cls, path=None, system=False, only_python=False, global_search=True): @@ -300,7 +302,7 @@ class PathEntry(BasePath): children = self.path.iterdir() return children - @property + @cached_property def children(self): if not self._children and self.is_dir and self.is_root: self._children = { @@ -308,7 +310,7 @@ class PathEntry(BasePath): for child in self._filter_children() } elif not self.is_dir: - return {self.path.as_posix(): self} + self._children = {self.path.as_posix(): self} return self._children @pythons.default @@ -325,7 +327,7 @@ class PathEntry(BasePath): pythons[_path.as_posix()] = copy.deepcopy(self) return pythons - @property + @cached_property def as_python(self): if not self.is_dir and self.is_python: if not self.py_version: @@ -371,11 +373,11 @@ class PathEntry(BasePath): _new._children = children return _new - @property + @cached_property def name(self): return self.path.name - @property + @cached_property def is_dir(self): try: ret_val = self.path.is_dir() @@ -383,11 +385,11 @@ class PathEntry(BasePath): ret_val = False return ret_val - @property + @cached_property def is_executable(self): return path_is_known_executable(self.path) - @property + @cached_property def is_python(self): return self.is_executable and ( self.py_version or looks_like_python(self.path.name) From dbcb5a71b389d7c3e31298297f3edf1842572cf6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 18 Jul 2018 00:16:17 -0400 Subject: [PATCH 055/101] Add cached property to vendored deps Signed-off-by: Dan Ryan --- pipenv/vendor/cached-property.LICENSE | 12 ++ pipenv/vendor/cached_property.py | 152 ++++++++++++++++++++++++++ pipenv/vendor/vendor.txt | 1 + tasks/vendoring/__init__.py | 96 +++++++++++----- 4 files changed, 235 insertions(+), 26 deletions(-) create mode 100644 pipenv/vendor/cached-property.LICENSE create mode 100644 pipenv/vendor/cached_property.py diff --git a/pipenv/vendor/cached-property.LICENSE b/pipenv/vendor/cached-property.LICENSE new file mode 100644 index 00000000..a181761c --- /dev/null +++ b/pipenv/vendor/cached-property.LICENSE @@ -0,0 +1,12 @@ +Copyright (c) 2015, Daniel Greenfeld +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +* Neither the name of cached-property nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/cached_property.py b/pipenv/vendor/cached_property.py new file mode 100644 index 00000000..a06be97a --- /dev/null +++ b/pipenv/vendor/cached_property.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- + +__author__ = "Daniel Greenfeld" +__email__ = "pydanny@gmail.com" +__version__ = "1.4.3" +__license__ = "BSD" + +from time import time +import threading + +try: + import asyncio +except (ImportError, SyntaxError): + asyncio = None + + +class cached_property(object): + """ + A property that is only computed once per instance and then replaces itself + with an ordinary attribute. Deleting the attribute resets the property. + Source: https://github.com/bottlepy/bottle/commit/fa7733e075da0d790d809aa3d2f53071897e6f76 + """ # noqa + + def __init__(self, func): + self.__doc__ = getattr(func, "__doc__") + self.func = func + + def __get__(self, obj, cls): + if obj is None: + return self + + if asyncio and asyncio.iscoroutinefunction(self.func): + return self._wrap_in_coroutine(obj) + + value = obj.__dict__[self.func.__name__] = self.func(obj) + return value + + def _wrap_in_coroutine(self, obj): + + @asyncio.coroutine + def wrapper(): + future = asyncio.ensure_future(self.func(obj)) + obj.__dict__[self.func.__name__] = future + return future + + return wrapper() + + +class threaded_cached_property(object): + """ + A cached_property version for use in environments where multiple threads + might concurrently try to access the property. + """ + + def __init__(self, func): + self.__doc__ = getattr(func, "__doc__") + self.func = func + self.lock = threading.RLock() + + def __get__(self, obj, cls): + if obj is None: + return self + + obj_dict = obj.__dict__ + name = self.func.__name__ + with self.lock: + try: + # check if the value was computed before the lock was acquired + return obj_dict[name] + + except KeyError: + # if not, do the calculation and release the lock + return obj_dict.setdefault(name, self.func(obj)) + + +class cached_property_with_ttl(object): + """ + A property that is only computed once per instance and then replaces itself + with an ordinary attribute. Setting the ttl to a number expresses how long + the property will last before being timed out. + """ + + def __init__(self, ttl=None): + if callable(ttl): + func = ttl + ttl = None + else: + func = None + self.ttl = ttl + self._prepare_func(func) + + def __call__(self, func): + self._prepare_func(func) + return self + + def __get__(self, obj, cls): + if obj is None: + return self + + now = time() + obj_dict = obj.__dict__ + name = self.__name__ + try: + value, last_updated = obj_dict[name] + except KeyError: + pass + else: + ttl_expired = self.ttl and self.ttl < now - last_updated + if not ttl_expired: + return value + + value = self.func(obj) + obj_dict[name] = (value, now) + return value + + def __delete__(self, obj): + obj.__dict__.pop(self.__name__, None) + + def __set__(self, obj, value): + obj.__dict__[self.__name__] = (value, time()) + + def _prepare_func(self, func): + self.func = func + if func: + self.__doc__ = func.__doc__ + self.__name__ = func.__name__ + self.__module__ = func.__module__ + + +# Aliases to make cached_property_with_ttl easier to use +cached_property_ttl = cached_property_with_ttl +timed_cached_property = cached_property_with_ttl + + +class threaded_cached_property_with_ttl(cached_property_with_ttl): + """ + A cached_property version for use in environments where multiple threads + might concurrently try to access the property. + """ + + def __init__(self, ttl=None): + super(threaded_cached_property_with_ttl, self).__init__(ttl) + self.lock = threading.RLock() + + def __get__(self, obj, cls): + with self.lock: + return super(threaded_cached_property_with_ttl, self).__get__(obj, cls) + + +# Alias to make threaded_cached_property_with_ttl easier to use +threaded_cached_property_ttl = threaded_cached_property_with_ttl +timed_threaded_cached_property = threaded_cached_property_with_ttl diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 961ad016..e2c3165c 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -39,3 +39,4 @@ six==1.11.0 semver==2.8.0 shutilwhich==1.1.0 toml==0.9.4 +cached-property==1.4.3 diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 9b8a2c82..a93360ce 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -280,19 +280,47 @@ def write_backport_imports(ctx, vendor_dir): backport_init.write_text('\n'.join(init_py_lines) + '\n') -def vendor(ctx, vendor_dir, rewrite=True): - log('Reinstalling vendored libraries') - is_patched = vendor_dir.name == 'patched' - requirements_file = vendor_dir.name +def _ensure_package_in_requirements(ctx, requirements_file, package): + requirement = None + log('using requirements file: %s' % requirements_file) + req_file_lines = [l for l in requirements_file.read_text().splitlines()] + if package: + match = [r for r in req_file_lines if r.strip().lower().startswith(package)] + matched_req = None + if match: + for m in match: + specifiers = [m.index(s) for s in ['>', '<', '=', '~'] if s in m] + if m.lower() == package or (specifiers and m[:min(specifiers)].lower() == package): + matched_req = "{0}".format(m) + requirement = matched_req + log("Matched req: %r" % matched_req) + if not matched_req: + req_file_lines.append("{0}".format(package)) + log("Writing requirements file: %s" % requirements_file) + requirements_file.write_text('\n'.join(req_file_lines)) + requirement = "{0}".format(package) + return requirement + + + +def install(ctx, vendor_dir, package=None): + requirements_file = vendor_dir / "{0}.txt".format(vendor_dir.name) + requirement = "-r {0}".format(requirements_file.as_posix()) + log('Using requirements file: %s' % requirement) + if package: + requirement = _ensure_package_in_requirements(ctx, requirements_file, package) # We use --no-deps because we want to ensure that all of our dependencies # are added to vendor.txt, this includes all dependencies recursively up # the chain. ctx.run( - 'pip install -t {0} -r {0}/{1}.txt --no-compile --no-deps'.format( - str(vendor_dir), - requirements_file, + 'pip install -t {0} --no-compile --no-deps {1}'.format( + vendor_dir.as_posix(), + requirement, ) ) + + +def post_install_cleanup(ctx, vendor_dir): remove_all(vendor_dir.glob('*.dist-info')) remove_all(vendor_dir.glob('*.egg-info')) @@ -300,6 +328,13 @@ def vendor(ctx, vendor_dir, rewrite=True): drop_dir(vendor_dir / 'bin') drop_dir(vendor_dir / 'tests') + +def vendor(ctx, vendor_dir, package=None, rewrite=True): + log('Reinstalling vendored libraries') + is_patched = vendor_dir.name == 'patched' + install(ctx, vendor_dir, package=package) + log('Running post-install cleanup...') + post_install_cleanup(ctx, vendor_dir) # Detect the vendored packages/modules vendored_libs = detect_vendored_libs(_get_vendor_dir(ctx)) patched_libs = detect_vendored_libs(_get_patched_dir(ctx)) @@ -320,25 +355,26 @@ def vendor(ctx, vendor_dir, rewrite=True): log('Renaming specified libs...') for item in vendor_dir.iterdir(): if item.is_dir(): - if rewrite: + if rewrite and not package or (package and item.name.lower() in package): log('Rewriting imports for %s...' % item) rewrite_imports(item, vendored_libs, vendor_dir) rename_if_needed(ctx, vendor_dir, item) elif item.name not in FILE_WHITE_LIST: - if rewrite: + if rewrite and not package or (package and item.stem.lower() in package): rewrite_file_imports(item, vendored_libs, vendor_dir) write_backport_imports(ctx, vendor_dir) - log('Applying post-patches...') - patches = patch_dir.glob('*.patch' if not is_patched else '_post*.patch') - for patch in patches: - apply_patch(ctx, patch) - if is_patched: - piptools_vendor = vendor_dir / 'piptools' / '_vendored' - if piptools_vendor.exists(): - drop_dir(piptools_vendor) - msgpack = vendor_dir / 'notpip' / '_vendor' / 'msgpack' - if msgpack.exists(): - remove_all(msgpack.glob('*.so')) + if not package: + log('Applying post-patches...') + patches = patch_dir.glob('*.patch' if not is_patched else '_post*.patch') + for patch in patches: + apply_patch(ctx, patch) + if is_patched: + piptools_vendor = vendor_dir / 'piptools' / '_vendored' + if piptools_vendor.exists(): + drop_dir(piptools_vendor) + msgpack = vendor_dir / 'notpip' / '_vendor' / 'msgpack' + if msgpack.exists(): + remove_all(msgpack.glob('*.so')) @invoke.task @@ -371,16 +407,19 @@ def rewrite_all_imports(ctx): @invoke.task -def download_licenses(ctx, vendor_dir, requirements_file='vendor.txt'): +def download_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', package=None): log('Downloading licenses') if not vendor_dir: vendor_dir = _get_vendor_dir(ctx) + requirements_file = vendor_dir / requirements_file + requirement = "-r {0}".format(requirements_file.as_posix()) + if package: + requirement = _ensure_package_in_requirements(ctx, requirements_file, package) tmp_dir = vendor_dir / '__tmp__' ctx.run( - 'pip download -r {0}/{1} --no-binary :all: --no-deps -d {2}'.format( - str(vendor_dir), - requirements_file, - str(tmp_dir), + 'pip download --no-binary :all: --no-deps -d {0} {1}'.format( + tmp_dir.as_posix(), + requirement, ) ) for sdist in tmp_dir.iterdir(): @@ -503,10 +542,15 @@ def generate_patch(ctx, package_path, patch_description, base='HEAD'): @invoke.task(name=TASK_NAME) -def main(ctx): +def main(ctx, package=None): vendor_dir = _get_vendor_dir(ctx) patched_dir = _get_patched_dir(ctx) log('Using vendor dir: %s' % vendor_dir) + if package: + vendor(ctx, vendor_dir, package=package) + download_licenses(ctx, vendor_dir, package=package) + log("Vendored %s" % package) + return clean_vendor(ctx, vendor_dir) clean_vendor(ctx, patched_dir) vendor(ctx, vendor_dir) From b8aa62ac664b8e679c1b7b52dce393aff42b02de Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 18 Jul 2018 14:47:04 +0800 Subject: [PATCH 056/101] Python --- pipenv/vendor/pythonfinder/models/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 4c6aa1f6..f0bcbfd0 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -69,7 +69,7 @@ class PythonVersion(object): ) def matches(self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None): - if arch and arch.isnumeric(): + if arch and arch.isdigit(): arch = '{0}bit'.format(arch) return ( (major is None or self.major == major) From 29326d53f6557785fba01a59e692ddf3c77772cd Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 18 Jul 2018 14:47:54 +0800 Subject: [PATCH 057/101] Block inside spinner block --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index c3bf6612..eaea6e41 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -847,7 +847,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): c = delegator.run( cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config, ) - c.block() + c.block() click.echo(crayons.blue("{0}".format(c.out)), err=True) if c.return_code != 0: click.echo(crayons.blue("{0}".format(c.err)), err=True) From c9277c500833e2f435d5258b1f82a10fbfc8561a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 18 Jul 2018 15:14:11 +0800 Subject: [PATCH 058/101] More isdigit() fixes --- pipenv/vendor/pythonfinder/models/python.py | 2 +- pipenv/vendor/pythonfinder/pythonfinder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index f0bcbfd0..fcb48723 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -196,7 +196,7 @@ class PythonVersion(object): @classmethod def create(cls, **kwargs): if 'architecture' in kwargs: - if kwargs['architecture'].isnumeric(): + if kwargs['architecture'].isdigit(): kwargs['architecture'] = '{0}bit'.format(kwargs['architecture']) return cls(**kwargs) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index fa8706f1..c6edbaf9 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -55,7 +55,7 @@ class Finder(object): if isinstance(major, six.string_types) and pre is None and minor is None and dev is None and patch is None: if arch is None and '-' in major: major, arch = major.rsplit('-', 1) - if not arch.isnumeric(): + if not arch.isdigit(): major = "{0}-{1}".format(major, arch) else: arch = "{0}bit".format(arch) From e3f37bebfe50d5248c3330adfd421be4ff93a155 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 26 Jul 2018 02:42:15 +0800 Subject: [PATCH 059/101] Fix missing arch arguments --- pipenv/vendor/pythonfinder/models/windows.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index 90f2d803..40dd93ed 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -19,7 +19,7 @@ class WindowsFinder(BaseFinder): def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -30,7 +30,7 @@ class WindowsFinder(BaseFinder): def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): return next(( v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch )), None ) From f9655cbacf5a33115c24e0d49a492ac4eb24c6ac Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 25 Jul 2018 16:56:49 -0400 Subject: [PATCH 060/101] Update vendoring tasks for updating single vendored deps Signed-off-by: Dan Ryan --- pipenv/vendor/vendor.txt | 1 + tasks/vendoring/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index e2c3165c..c4f44123 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -40,3 +40,4 @@ semver==2.8.0 shutilwhich==1.1.0 toml==0.9.4 cached-property==1.4.3 +pythonfinder==1.0.0 \ No newline at end of file diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index a93360ce..4deb58b8 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -313,7 +313,7 @@ def install(ctx, vendor_dir, package=None): # are added to vendor.txt, this includes all dependencies recursively up # the chain. ctx.run( - 'pip install -t {0} --no-compile --no-deps {1}'.format( + 'pip install -t {0} --no-compile --no-deps --upgrade {1}'.format( vendor_dir.as_posix(), requirement, ) From f5df34f85bbb64237b1ce6130a05a14f7e815453 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 25 Jul 2018 17:02:32 -0400 Subject: [PATCH 061/101] Update vendoring instructions Signed-off-by: Dan Ryan --- pipenv/vendor/README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/pipenv/vendor/README.md b/pipenv/vendor/README.md index 11bbbb4b..0e3ab8bb 100644 --- a/pipenv/vendor/README.md +++ b/pipenv/vendor/README.md @@ -2,10 +2,21 @@ These packages are copied as-is from upstream to reduce Pipenv dependencies. They should always be kept synced with upstream. DO NOT MODIFY DIRECTLY! If -you need to patch anything, move the package to `patched`. +you need to patch anything, move the package to `patched` and generate a +patch for it using `git diff -p `. This patch belongs +in `./pipenv/tasks/vendoring/patches/patched/.patch`. -Known vendored versions: +To add a vendored dependency or to update a single dependency, use the +vendoring scripts: +``` + pipenv run inv vendoring.update --package="pkgname==versionnum" +``` -- python-dotenv: 0.8.2 +This will automatically pin the package in `./pipenv/vendor/vendor.txt` +or it will update the pin if the package is already present, and it will +then update the package and download any necessary licenses (if available). +Note that this will not download any dependencies, you must add those each +individually. -When updating, update the corresponding LICENSE files as well. +When updating, ensure that the corresponding LICENSE files are still +up-to-date. From ef060c4b9b5c119e60feaba167bb22a273e04d90 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 25 Jul 2018 17:02:45 -0400 Subject: [PATCH 062/101] Update pythonfinder Signed-off-by: Dan Ryan --- news/2582.bugfix | 1 + news/2582.feature | 4 + news/2582.vendor | 1 + pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/_vendor/Makefile | 14 ---- pipenv/vendor/pythonfinder/_vendor/vendor.txt | 1 - pipenv/vendor/pythonfinder/exceptions.py | 1 + pipenv/vendor/pythonfinder/models/__init__.py | 43 +++++++++-- pipenv/vendor/pythonfinder/models/path.py | 73 ++++++++++++------- pipenv/vendor/pythonfinder/models/pyenv.py | 4 +- pipenv/vendor/pythonfinder/models/python.py | 26 +++---- pipenv/vendor/pythonfinder/models/windows.py | 28 +++++-- pipenv/vendor/pythonfinder/pythonfinder.py | 48 ++++++++---- pipenv/vendor/pythonfinder/utils.py | 5 +- 14 files changed, 163 insertions(+), 88 deletions(-) create mode 100644 news/2582.bugfix create mode 100644 news/2582.feature create mode 100644 news/2582.vendor delete mode 100644 pipenv/vendor/pythonfinder/_vendor/Makefile delete mode 100644 pipenv/vendor/pythonfinder/_vendor/vendor.txt diff --git a/news/2582.bugfix b/news/2582.bugfix new file mode 100644 index 00000000..f434be81 --- /dev/null +++ b/news/2582.bugfix @@ -0,0 +1 @@ +Fixed multiple issues with finding the correct system python locations. diff --git a/news/2582.feature b/news/2582.feature new file mode 100644 index 00000000..4d1f7a36 --- /dev/null +++ b/news/2582.feature @@ -0,0 +1,4 @@ +Greatly enhanced python discovery functionality: + +- Added pep514 (windows launcher/finder) support for python discovery. +- Introduced architecture discovery for python installations which support different architectures. diff --git a/news/2582.vendor b/news/2582.vendor new file mode 100644 index 00000000..1a031eb7 --- /dev/null +++ b/news/2582.vendor @@ -0,0 +1 @@ +Update ``pythonfinder`` to major release ``1.0.0`` for integration. diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index eb3d0363..9f1628be 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = "0.1.4.dev0" +__version__ = "1.0.0" __all__ = ["Finder", "WindowsFinder", "SystemPath", "InvalidPythonVersion"] from .pythonfinder import Finder diff --git a/pipenv/vendor/pythonfinder/_vendor/Makefile b/pipenv/vendor/pythonfinder/_vendor/Makefile deleted file mode 100644 index 5c44fea4..00000000 --- a/pipenv/vendor/pythonfinder/_vendor/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -# Taken from pip: https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/src/pip/_vendor/Makefile -all: clean vendor - -clean: - @# Delete vendored items - find . -maxdepth 1 -mindepth 1 -type d -exec rm -rf {} \; - -vendor: - @# Install vendored libraries - pip install -t . -r vendor.txt - - @# Cleanup .egg-info directories - rm -rf *.egg-info - rm -rf *.dist-info diff --git a/pipenv/vendor/pythonfinder/_vendor/vendor.txt b/pipenv/vendor/pythonfinder/_vendor/vendor.txt deleted file mode 100644 index 88752498..00000000 --- a/pipenv/vendor/pythonfinder/_vendor/vendor.txt +++ /dev/null @@ -1 +0,0 @@ --e git+https://github.com/zooba/pep514tools.git@320e48745660b696e2dcaee888fc2e516b435e48#egg=pep514tools diff --git a/pipenv/vendor/pythonfinder/exceptions.py b/pipenv/vendor/pythonfinder/exceptions.py index 13e56e2c..df381daf 100644 --- a/pipenv/vendor/pythonfinder/exceptions.py +++ b/pipenv/vendor/pythonfinder/exceptions.py @@ -4,4 +4,5 @@ from __future__ import print_function, absolute_import class InvalidPythonVersion(Exception): """Raised when parsing an invalid python version""" + pass diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index 7cf0fadf..a38494ed 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -40,10 +40,19 @@ class BasePath(object): for ext in KNOWN_EXTS ] children = self.children - found = next((children[(self.path / child).as_posix()] for child in valid_names if (self.path / child).as_posix() in children), None) + found = next( + ( + children[(self.path / child).as_posix()] + for child in valid_names + if (self.path / child).as_posix() in children + ), + None, + ) return found - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -57,7 +66,9 @@ class BasePath(object): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - call_method = "find_all_python_versions" if self.is_dir else "find_python_version" + call_method = ( + "find_all_python_versions" if self.is_dir else "find_python_version" + ) sub_finder = operator.methodcaller( call_method, major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) @@ -67,7 +78,9 @@ class BasePath(object): version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_python_version( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -81,7 +94,13 @@ class BasePath(object): """ version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") @@ -89,13 +108,23 @@ class BasePath(object): if self.is_python and self.as_python and version_matcher(self.as_python): return self return - finder = ((child, child.as_python) for child in unnest(self.pythons.values()) if child.as_python) + finder = ( + (child, child.as_python) + for child in unnest(self.pythons.values()) + if child.as_python + ) py_filter = filter( None, filter(lambda child: version_matcher(child[1]), finder) ) version_sort = operator.attrgetter("version_sort") return next( - (c[0] for c in sorted(py_filter, key=lambda child: child[1].version_sort, reverse=True)), None + ( + c[0] + for c in sorted( + py_filter, key=lambda child: child[1].version_sort, reverse=True + ) + ), + None, ) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 6857d17d..af103915 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -49,7 +49,11 @@ class SystemPath(object): @cached_property def executables(self): - self.executables = [p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable] + self.executables = [ + p + for p in chain(*(child.children.values() for child in self.paths.values())) + if p.is_executable + ] return self.executables @cached_property @@ -69,7 +73,7 @@ class SystemPath(object): self._version_dict = defaultdict(list) for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): - if finder_name == 'windows': + if finder_name == "windows": if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue @@ -98,17 +102,15 @@ class SystemPath(object): self._setup_windows() if PYENV_INSTALLED: self._setup_pyenv() - venv = os.environ.get('VIRTUAL_ENV') - if os.name == 'nt': - bin_dir = 'Scripts' + venv = os.environ.get("VIRTUAL_ENV") + if os.name == "nt": + bin_dir = "Scripts" else: - bin_dir = 'bin' + bin_dir = "bin" if venv and (self.system or self.global_search): p = ensure_path(venv) self.path_order = [(p / bin_dir).as_posix()] + self.path_order - self.paths[p] = PathEntry.create( - path=p, is_root=True, only_python=False - ) + self.paths[p] = PathEntry.create(path=p, is_root=True, only_python=False) if self.system: syspath = Path(sys.executable) syspath_bin = syspath.parent @@ -141,7 +143,7 @@ class SystemPath(object): before_path + [p.path.as_posix() for p in root_paths] + after_path ) self.paths.update({p.path: p for p in root_paths}) - self._register_finder('pyenv', self.pyenv_finder) + self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): from .windows import WindowsFinder @@ -151,13 +153,13 @@ class SystemPath(object): path_addition = [p.path.as_posix() for p in root_paths] self.path_order = self.path_order[:] + path_addition self.paths.update({p.path: p for p in root_paths}) - self._register_finder('windows', self.windows_finder) + self._register_finder("windows", self.windows_finder) def get_path(self, path): path = ensure_path(path) _path = self.paths.get(path.as_posix()) if not _path and path.as_posix() in self.path_order: - _path = PathEntry.create( + _path = PathEntry.create( path=path.absolute(), is_root=True, only_python=self.only_python ) self.paths[path.as_posix()] = _path @@ -185,7 +187,9 @@ class SystemPath(object): filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) return next((f for f in filtered), None) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -200,18 +204,28 @@ class SystemPath(object): """ sub_finder = operator.methodcaller( - "find_all_python_versions", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "find_all_python_versions", + major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version paths = (self.get_path(k) for k in self.path_order) - path_filter = filter(None, unnest((sub_finder(p) for p in paths if p is not None))) + path_filter = filter( + None, unnest((sub_finder(p) for p in paths if p is not None)) + ) version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_python_version( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search for a specific python version on the path. :param major: Major python version to search for. @@ -226,7 +240,13 @@ class SystemPath(object): """ sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "find_python_version", + major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) if major and minor and patch: _tuple_pre = pre if pre is not None else False @@ -247,7 +267,7 @@ class SystemPath(object): if ver.as_python.version_tuple[:5] in self.python_version_dict: self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) else: - self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver,] + self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver] return ver @classmethod @@ -280,7 +300,13 @@ class SystemPath(object): for p in _path_objects } ) - return cls(paths=path_entries, path_order=paths, only_python=only_python, system=system, global_search=global_search) + return cls( + paths=path_entries, + path_order=paths, + only_python=only_python, + system=system, + global_search=global_search, + ) @attr.s @@ -293,7 +319,7 @@ class PathEntry(BasePath): pythons = attr.ib() def __str__(self): - return fs_str('{0}'.format(self.path.as_posix())) + return fs_str("{0}".format(self.path.as_posix())) def _filter_children(self): if self.only_python: @@ -333,6 +359,7 @@ class PathEntry(BasePath): if not self.py_version: try: from .python import PythonVersion + self.py_version = PythonVersion.from_path(self.path) except (ValueError, InvalidPythonVersion): self.py_version = None @@ -355,11 +382,7 @@ class PathEntry(BasePath): """ target = ensure_path(path) - creation_args = { - "path": target, - "is_root": is_root, - "only_python": only_python - } + creation_args = {"path": target, "is_root": is_root, "only_python": only_python} if pythons: creation_args["pythons"] = pythons _new = cls(**creation_args) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 13476b6b..6c890936 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -32,7 +32,9 @@ class PyenvFinder(BaseFinder): version.get("is_prerelease"), version.get("is_devrelease"), ) - versions[version_tuple] = VersionPath.create(path=p.resolve(), only_python=True) + versions[version_tuple] = VersionPath.create( + path=p.resolve(), only_python=True + ) return versions @pythons.default diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index fcb48723..6176fad4 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -44,13 +44,7 @@ class PythonVersion(object): release_sort = 1 elif self.is_devrelease: release_sort = 0 - return ( - self.major, - self.minor, - self.patch if self.patch else 0, - release_sort - ) - + return (self.major, self.minor, self.patch if self.patch else 0, release_sort) @property def version_tuple(self): @@ -68,9 +62,11 @@ class PythonVersion(object): self.is_devrelease, ) - def matches(self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None): - if arch and arch.isdigit(): - arch = '{0}bit'.format(arch) + def matches( + self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None + ): + if arch and arch.isnumeric(): + arch = "{0}bit".format(arch) return ( (major is None or self.major == major) and (minor is None or self.minor == minor) @@ -195,9 +191,9 @@ class PythonVersion(object): @classmethod def create(cls, **kwargs): - if 'architecture' in kwargs: - if kwargs['architecture'].isdigit(): - kwargs['architecture'] = '{0}bit'.format(kwargs['architecture']) + if "architecture" in kwargs: + if kwargs["architecture"].isnumeric(): + kwargs["architecture"] = "{0}bit".format(kwargs["architecture"]) return cls(**kwargs) @@ -221,4 +217,6 @@ class VersionMap(object): current_entries = {p.path for p in self.versions.get(version)} new_entries = {p.path for p in entries} new_entries -= current_entries - self.versions[version].append([e for e in entries if e.path in new_entries]) + self.versions[version].append( + [e for e in entries if e.path in new_entries] + ) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index 40dd93ed..f731432c 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -17,9 +17,17 @@ class WindowsFinder(BaseFinder): versions = attr.ib() pythons = attr.ib() - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -27,11 +35,17 @@ class WindowsFinder(BaseFinder): version_sort = operator.attrgetter("version_sort") return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): - return next(( - v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch - )), None + def find_python_version( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): + return next( + ( + v + for v in self.find_all_python_versions( + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + ) + ), + None, ) @versions.default diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index c6edbaf9..d6f49bf7 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -35,7 +35,9 @@ class Finder(object): def system_path(self): if not self._system_path: self._system_path = SystemPath.create( - path=self.path_prepend, system=self.system, global_search=self.global_search + path=self.path_prepend, + system=self.system, + global_search=self.global_search, ) return self._system_path @@ -50,12 +52,21 @@ class Finder(object): def which(self, exe): return self.system_path.which(exe) - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_python_version( + self, major, minor=None, patch=None, pre=None, dev=None, arch=None + ): from .models import PythonVersion - if isinstance(major, six.string_types) and pre is None and minor is None and dev is None and patch is None: - if arch is None and '-' in major: - major, arch = major.rsplit('-', 1) - if not arch.isdigit(): + + if ( + isinstance(major, six.string_types) + and pre is None + and minor is None + and dev is None + and patch is None + ): + if arch is None and "-" in major: + major, arch = major.rsplit("-", 1) + if not arch.isnumeric(): major = "{0}-{1}".format(major, arch) else: arch = "{0}bit".format(arch) @@ -76,19 +87,28 @@ class Finder(object): major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): version_sort = operator.attrgetter("as_python.version_sort") - python_version_dict = getattr(self.system_path, 'python_version_dict') + python_version_dict = getattr(self.system_path, "python_version_dict") if python_version_dict: - paths = filter(None, [path for version in python_version_dict.values() for path in version if path.as_python]) + paths = filter( + None, + [ + path + for version in python_version_dict.values() + for path in version + if path.as_python + ], + ) paths = sorted(paths, key=version_sort, reverse=True) return paths - versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) + versions = self.system_path.find_all_python_versions( + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + ) if not isinstance(versions, list): - versions = [versions,] - # if os.name == 'nt': - # windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) - # versions = list(windows_versions) + versions + versions = [versions] paths = sorted(versions, key=version_sort, reverse=True) path_map = {} for path in paths: diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index fdc54381..1a9bfa2c 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -32,10 +32,7 @@ def _run(cmd): """ encoding = locale.getdefaultlocale()[1] or "utf-8" c = subprocess.Popen( - cmd, - env=os.environ.copy(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + cmd, env=os.environ.copy(), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) out, err = c.communicate() return out.decode(encoding).strip(), err.decode(encoding).strip() From 5ff8bdeed3b58915a6aca63b061a424783ffa715 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 25 Jul 2018 17:23:28 -0400 Subject: [PATCH 063/101] Update vendor.txt Signed-off-by: Dan Ryan --- pipenv/vendor/vendor.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index c4f44123..47253dd3 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -21,7 +21,7 @@ git+https://github.com/naiquevin/pipdeptree.git@ee5eaf86ed0f49ea97601475e048d81e pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder +pythonfinder==1.0.0 requests==2.19.1 chardet==3.0.4 idna==2.7 @@ -40,4 +40,3 @@ semver==2.8.0 shutilwhich==1.1.0 toml==0.9.4 cached-property==1.4.3 -pythonfinder==1.0.0 \ No newline at end of file From ed8e862d296293d544aada35284e6cf776477d04 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 26 Jul 2018 14:04:55 +0800 Subject: [PATCH 064/101] isnumeric -> isdigit --- pipenv/vendor/pythonfinder/models/python.py | 4 ++-- pipenv/vendor/pythonfinder/pythonfinder.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 6176fad4..f10ddb4e 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -65,7 +65,7 @@ class PythonVersion(object): def matches( self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None ): - if arch and arch.isnumeric(): + if arch and arch.isdigit(): arch = "{0}bit".format(arch) return ( (major is None or self.major == major) @@ -192,7 +192,7 @@ class PythonVersion(object): @classmethod def create(cls, **kwargs): if "architecture" in kwargs: - if kwargs["architecture"].isnumeric(): + if kwargs["architecture"].isdigit(): kwargs["architecture"] = "{0}bit".format(kwargs["architecture"]) return cls(**kwargs) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index d6f49bf7..035842e2 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -66,7 +66,7 @@ class Finder(object): ): if arch is None and "-" in major: major, arch = major.rsplit("-", 1) - if not arch.isnumeric(): + if not arch.isdigit(): major = "{0}-{1}".format(major, arch) else: arch = "{0}bit".format(arch) From 059dacdd3c45e08d64792b7704c261f775fb2e88 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 26 Jul 2018 17:04:59 +0800 Subject: [PATCH 065/101] Use COMSPEC for shell detection fallback The subshell logic now considers: * PIPENV_SHELL * Auto detection * SHELL (should very likely work for POSIX) * PYENV_SHELL * COMSPEC (should always work for Windows) --- pipenv/environments.py | 6 +++++- pipenv/shells.py | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pipenv/environments.py b/pipenv/environments.py index afe82988..6e0c8651 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -207,7 +207,11 @@ if "PIPENV_ACTIVE" not in os.environ and not PIPENV_IGNORE_VIRTUALENVS: PIPENV_SKIP_VALIDATION = True # Internal, the default shell to use if shell detection fails. -PIPENV_SHELL = os.environ.get("SHELL") or os.environ.get("PYENV_SHELL") +PIPENV_SHELL = ( + os.environ.get("SHELL") or + os.environ.get("PYENV_SHELL") or + os.environ.get("COMSPEC") +) # Internal, to tell if pyenv is installed. PYENV_ROOT = os.environ.get("PYENV_ROOT", os.path.expanduser("~/.pyenv")) diff --git a/pipenv/shells.py b/pipenv/shells.py index 10826f22..70b236ea 100644 --- a/pipenv/shells.py +++ b/pipenv/shells.py @@ -29,7 +29,7 @@ def detect_info(): raise ShellDetectionFailure -def _get_activate_script(venv): +def _get_activate_script(cmd, venv): """Returns the string to activate a virtualenv. This is POSIX-only at the moment since the compat (pexpect-based) shell @@ -37,11 +37,11 @@ def _get_activate_script(venv): """ # Suffix and source command for other shells. # Support for fish shell. - if PIPENV_SHELL and "fish" in PIPENV_SHELL: + if "fish" in cmd: suffix = ".fish" command = "source" # Support for csh shell. - elif PIPENV_SHELL and "csh" in PIPENV_SHELL: + elif "csh" in cmd: suffix = ".csh" command = "source" else: @@ -104,7 +104,7 @@ class Shell(object): dims = get_terminal_size() with temp_environ(): c = pexpect.spawn(self.cmd, ["-i"], dimensions=(dims.lines, dims.columns)) - c.sendline(_get_activate_script(venv)) + c.sendline(_get_activate_script(self.cmd, venv)) if args: c.sendline(" ".join(args)) From 553ed8258c7f217b4e5f61fa7182a239b2dc7fe9 Mon Sep 17 00:00:00 2001 From: Louie Lu Date: Mon, 23 Jul 2018 20:29:30 +0800 Subject: [PATCH 066/101] Add JSONDecodeError handling for get_lockfile_hash Now will return None when facing JSONDecodeError when loading the lockfile. do_init will remove the old lockfile and replace it. --- pipenv/core.py | 20 ++++++++++++++++++++ pipenv/project.py | 6 +++++- tests/integration/test_lock.py | 22 ++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 25bd0b7e..d72d2e0f 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1257,6 +1257,26 @@ def do_init( ), err=True, ) + elif old_hash is None: + # Lockfile corrupted, remove it and replaced + click.echo( + crayons.red( + u"Pipfile.lock is corrupted, replaced with ({0})…".format( + new_hash[-6:] + ), + bold=True, + ), + err=True + ) + os.remove(project.lockfile_location) + do_lock( + system=system, + pre=pre, + keep_outdated=keep_outdated, + verbose=verbose, + write=True, + pypi_mirror=pypi_mirror, + ) else: click.echo( crayons.red( diff --git a/pipenv/project.py b/pipenv/project.py index b12f7c64..275a6b41 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -815,7 +815,11 @@ class Project(object): if not os.path.exists(self.lockfile_location): return - lockfile = self.load_lockfile(expand_env_vars=False) + try: + lockfile = self.load_lockfile(expand_env_vars=False) + except json.decoder.JSONDecodeError: + # Lockfile corrupted + return if "_meta" in lockfile and hasattr(lockfile, "keys"): return lockfile["_meta"].get("hash", {}).get("sha256") # Lockfile exists but has no hash at all diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 25195bfb..a0cdfe15 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -385,3 +385,25 @@ django = "*" django_version = '==2.0.6' if py_version.startswith('3') else '==1.11.13' assert py_version == '2.7.14' assert p.lockfile['default']['django']['version'] == django_version + + +@pytest.mark.lock +@pytest.mark.install +def test_lockfile_corrupted(PipenvInstance): + with PipenvInstance() as p: + with open(p.lockfile_path, 'w') as f: + f.write('{corrupt}') + c = p.pipenv('install') + assert c.return_code == 0 + assert p.lockfile['_meta'] + + +@pytest.mark.lock +@pytest.mark.install +def test_lockfile_with_empty_dict(PipenvInstance): + with PipenvInstance() as p: + with open(p.lockfile_path, 'w') as f: + f.write('{}') + c = p.pipenv('install') + assert c.return_code == 0 + assert p.lockfile['_meta'] \ No newline at end of file From 79349dd8f1224d9235c5b9bf8d66bfcc6067e753 Mon Sep 17 00:00:00 2001 From: Louie Lu Date: Tue, 24 Jul 2018 09:45:12 +0800 Subject: [PATCH 067/101] Addressed uranusjr comments --- pipenv/core.py | 50 +++++++++++++++------------------- pipenv/project.py | 4 +-- tests/integration/test_lock.py | 6 ++-- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index d72d2e0f..c1e7d015 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1257,18 +1257,29 @@ def do_init( ), err=True, ) - elif old_hash is None: - # Lockfile corrupted, remove it and replaced - click.echo( - crayons.red( - u"Pipfile.lock is corrupted, replaced with ({0})…".format( - new_hash[-6:] + else: + if not old_hash: + # Lockfile corrupted, remove it and replaced + click.echo( + crayons.red( + u"Pipfile.lock is corrupted, replaced with ({0})…".format( + new_hash[-6:] + ), + bold=True, ), - bold=True, - ), - err=True - ) - os.remove(project.lockfile_location) + err=True + ) + os.remove(project.lockfile_location) + else: + click.echo( + crayons.red( + u"Pipfile.lock ({0}) out of date, updating to ({1})…".format( + old_hash[-6:], new_hash[-6:] + ), + bold=True, + ), + err=True, + ) do_lock( system=system, pre=pre, @@ -1277,23 +1288,6 @@ def do_init( write=True, pypi_mirror=pypi_mirror, ) - else: - click.echo( - crayons.red( - u"Pipfile.lock ({0}) out of date, updating to ({1})…".format( - old_hash[-6:], new_hash[-6:] - ), - bold=True, - ), - err=True, - ) - do_lock( - system=system, - pre=pre, - keep_outdated=keep_outdated, - write=True, - pypi_mirror=pypi_mirror, - ) # Write out the lockfile if it doesn't exist. if not project.lockfile_exists and not skip_lock: # Unless we're in a virtualenv not managed by pipenv, abort if we're diff --git a/pipenv/project.py b/pipenv/project.py index 275a6b41..c5f451e4 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -817,9 +817,9 @@ class Project(object): try: lockfile = self.load_lockfile(expand_env_vars=False) - except json.decoder.JSONDecodeError: + except ValueError: # Lockfile corrupted - return + return "" if "_meta" in lockfile and hasattr(lockfile, "keys"): return lockfile["_meta"].get("hash", {}).get("sha256") # Lockfile exists but has no hash at all diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index a0cdfe15..23d75f8e 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -392,9 +392,10 @@ django = "*" def test_lockfile_corrupted(PipenvInstance): with PipenvInstance() as p: with open(p.lockfile_path, 'w') as f: - f.write('{corrupt}') + f.write('{corrupted}') c = p.pipenv('install') assert c.return_code == 0 + assert 'Pipfile.lock is corrupted' in c.err assert p.lockfile['_meta'] @@ -406,4 +407,5 @@ def test_lockfile_with_empty_dict(PipenvInstance): f.write('{}') c = p.pipenv('install') assert c.return_code == 0 - assert p.lockfile['_meta'] \ No newline at end of file + assert 'Pipfile.lock is corrupted' in c.err + assert p.lockfile['_meta'] From bd131836cd95193a5d2fa0ec0873079f622594dc Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 30 Jul 2018 12:26:07 +0800 Subject: [PATCH 068/101] Don't remove corrupt lock file --- pipenv/core.py | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index c1e7d015..2bb9140e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1258,28 +1258,14 @@ def do_init( err=True, ) else: - if not old_hash: - # Lockfile corrupted, remove it and replaced - click.echo( - crayons.red( - u"Pipfile.lock is corrupted, replaced with ({0})…".format( - new_hash[-6:] - ), - bold=True, - ), - err=True - ) - os.remove(project.lockfile_location) + if old_hash: + msg = u"Pipfile.lock ({1}) out of date, updating to ({0})…" else: - click.echo( - crayons.red( - u"Pipfile.lock ({0}) out of date, updating to ({1})…".format( - old_hash[-6:], new_hash[-6:] - ), - bold=True, - ), - err=True, - ) + msg = u"Pipfile.lock is corrupted, replaced with ({0})…" + click.echo(crayons.red( + msg.format(old_hash[-6:], new_hash[-6:]), + bold=True, + ), err=True) do_lock( system=system, pre=pre, From a67d0244a046a0a8efbd16d7ac37b611bd9145a1 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 30 Jul 2018 12:49:28 +0800 Subject: [PATCH 069/101] News --- news/2607.bugfix | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/2607.bugfix diff --git a/news/2607.bugfix b/news/2607.bugfix new file mode 100644 index 00000000..101124a8 --- /dev/null +++ b/news/2607.bugfix @@ -0,0 +1,2 @@ +Catch JSON decoding error to prevent exception when the lock file is of +invalid format. From 7758d745a8bed0c1da9fba31745378e58732029f Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 30 Jul 2018 12:51:25 +0800 Subject: [PATCH 070/101] News --- news/2651.behavior | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/2651.behavior diff --git a/news/2651.behavior b/news/2651.behavior new file mode 100644 index 00000000..be5f2d01 --- /dev/null +++ b/news/2651.behavior @@ -0,0 +1,2 @@ +Add ``COMSPEC`` to fallback option (along with ``SHELL`` and ``PYENV_SHELL``) +if shell detection fails, improving robustness on Windows. From 2db196dc2685ba5e3b3130ffcc27616f27da71b3 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 30 Jul 2018 14:51:23 +0800 Subject: [PATCH 071/101] PR template https://help.github.com/articles/creating-a-pull-request-template-for-your-repository/ --- .github/PULL_REQUEST_TEMPLATE.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..911d83b3 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,28 @@ +Thank you for contributing to Pipenv! + + +##### The issue + +What is the thing you want to fix? Is it associated with an issue on GitHub? Please mention it. + +Always consider opening an issue first to describe your problem, so we can discuss what is the best way to amend it. + + +##### The fix + +How does this pull request fix your problem? Did you consider any alternatives? Why is this the *best* solution, in your opinion? + + +##### The checklist + +* [ ] Associated issue +* [ ] A news fragment in the `news/` directory to describe this fix (this will appear in the release changelog). + + +##### If this is a patch to the `vendor` directory… + +Please try to refrain from submitting patches directly to `vendor` or `patched`, but raise your issue to the upstream project instead, and inform Pipenv to upgrade when the upstream project accepts the fix. + +A pull request to upgrade vendor packages is strongly discouraged, unless there is a very good reason (e.g. you need to test Pipenv’s integration to a new vendor feature). Pipenv audits and perform vendor upgrades regularly, generally before a new release is about to drop. + +If your patch is not or cannot be accepted by upstream, but is essential to Pipenv (make sure to discuss this with maintainers!), please remember to attach a patch file in `tasks/vendoring/patched`, so this divergence from upstream can be recorded and replayed afterwards. From 386340bead35e2480cff97c1b79c44a93a2db623 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 15:45:43 +0800 Subject: [PATCH 072/101] Unify verbosity flags and environment variable --- pipenv/cli.py | 69 +++++++++++++++++++++++++++----------------------- pipenv/core.py | 15 +++-------- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 1b774508..49722dae 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -61,12 +61,12 @@ class PipenvGroup(Group): ) -def setup_verbose(ctx, param, value): - if value: - import logging - - logging.getLogger("pip").setLevel(logging.INFO) - return value +def setup_verbosity(ctx, param, value): + if not value: + return + import logging + logging.getLogger("pip").setLevel(logging.INFO) + environments.PIPENV_VERBOSITY = 1 def validate_python_path(ctx, param, value): @@ -333,9 +333,9 @@ def cli( "--verbose", "-v", is_flag=True, - default=False, + expose_value=False, + callback=setup_verbosity, help="Verbose mode.", - callback=setup_verbose, ) @option( "--ignore-pipfile", @@ -385,7 +385,6 @@ def install( lock=True, ignore_pipfile=False, skip_lock=False, - verbose=False, requirements=False, sequential=False, pre=False, @@ -408,7 +407,7 @@ def install( lock=lock, ignore_pipfile=ignore_pipfile, skip_lock=skip_lock, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), requirements=requirements, sequential=sequential, pre=pre, @@ -440,9 +439,9 @@ def install( "--verbose", "-v", is_flag=True, - default=False, + expose_value=False, help="Verbose mode.", - callback=setup_verbose, + callback=setup_verbosity, ) @option("--lock", is_flag=True, default=True, help="Lock afterwards.") @option( @@ -479,7 +478,6 @@ def uninstall( lock=False, all_dev=False, all=False, - verbose=False, keep_outdated=False, pypi_mirror=None, ): @@ -495,7 +493,7 @@ def uninstall( lock=lock, all_dev=all_dev, all=all, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), keep_outdated=keep_outdated, pypi_mirror=pypi_mirror, ) @@ -526,9 +524,9 @@ def uninstall( "--verbose", "-v", is_flag=True, - default=False, + expose_value=False, help="Verbose mode.", - callback=setup_verbose, + callback=setup_verbosity, ) @option( "--requirements", @@ -556,7 +554,6 @@ def lock( three=None, python=False, pypi_mirror=None, - verbose=False, requirements=False, dev=False, clear=False, @@ -571,7 +568,7 @@ def lock( if requirements: do_init(dev=dev, requirements=requirements, pypi_mirror=pypi_mirror) do_lock( - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), clear=clear, pre=pre, keep_outdated=keep_outdated, @@ -783,9 +780,9 @@ def check( "--verbose", "-v", is_flag=True, - default=False, + expose_value=False, help="Verbose mode.", - callback=setup_verbose, + callback=setup_verbosity, ) @option( "--dev", @@ -821,7 +818,6 @@ def update( python=False, pypi_mirror=None, system=False, - verbose=False, clear=False, keep_outdated=False, pre=False, @@ -869,6 +865,8 @@ def update( err=True, ) sys.exit(1) + + verbose = (environments.PIPENV_VERBOSITY > 0) do_lock( verbose=verbose, clear=clear, @@ -954,9 +952,9 @@ def run_open(module, three=None, python=None, pypi_mirror=None): "--verbose", "-v", is_flag=True, - default=False, + expose_value=False, help="Verbose mode.", - callback=setup_verbose, + callback=setup_verbosity, ) @option( "--dev", @@ -1002,7 +1000,6 @@ def sync( bare=False, dont_upgrade=False, user=False, - verbose=False, clear=False, unused=False, package_name=None, @@ -1020,7 +1017,7 @@ def sync( bare=bare, dont_upgrade=dont_upgrade, user=user, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), clear=clear, unused=unused, sequential=sequential, @@ -1033,9 +1030,9 @@ def sync( "--verbose", "-v", is_flag=True, - default=False, + expose_value=False, help="Verbose mode.", - callback=setup_verbose, + callback=setup_verbosity, ) @option( "--three/--two", @@ -1050,15 +1047,23 @@ def sync( callback=validate_python_path, help="Specify which version of Python virtualenv should use.", ) -@option("--dry-run", is_flag=True, default=False, help="Just output unneeded packages.") +@option( + "--dry-run", + is_flag=True, + default=False, + help="Just output unneeded packages.", +) @pass_context -def clean( - ctx, three=None, python=None, dry_run=False, bare=False, user=False, verbose=False -): +def clean(ctx, three=None, python=None, dry_run=False, bare=False, user=False): """Uninstalls all packages not specified in Pipfile.lock.""" from .core import do_clean - do_clean(ctx=ctx, three=three, python=python, dry_run=dry_run, verbose=verbose) + do_clean( + ctx=ctx, + three=three, python=python, + dry_run=dry_run, + verbose=(environments.PIPENV_VERBOSITY > 0), + ) # Install click commands. diff --git a/pipenv/core.py b/pipenv/core.py index f22616ce..57db1325 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -41,7 +41,7 @@ from .utils import ( clean_resolved_dep, ) from ._compat import TemporaryDirectory, Path -from . import pep508checker, progress +from . import environments, pep508checker, progress from .environments import ( PIPENV_COLORBLIND, PIPENV_NOSPIN, @@ -49,7 +49,6 @@ from .environments import ( PIPENV_TIMEOUT, PIPENV_SKIP_VALIDATION, PIPENV_HIDE_EMOJIS, - PIPENV_INSTALL_TIMEOUT, PIPENV_YES, PIPENV_DONT_LOAD_ENV, PIPENV_DEFAULT_PYTHON_VERSION, @@ -1537,18 +1536,10 @@ def format_pip_output(out, r=None): def warn_in_virtualenv(): - from .environments import ( - PIPENV_USE_SYSTEM, - PIPENV_VIRTUALENV, - PIPENV_VERBOSITY, - ) - # Only warn if pipenv isn't already active. pipenv_active = os.environ.get("PIPENV_ACTIVE") - if ( - (PIPENV_USE_SYSTEM or PIPENV_VIRTUALENV) - and not (pipenv_active or PIPENV_VERBOSITY < 0) - ): + if ((environments.PIPENV_USE_SYSTEM or environments.PIPENV_VIRTUALENV) + and not (pipenv_active or environments.PIPENV_VERBOSITY < 0)): click.echo( "{0}: Pipenv found itself running within a virtual environment, " "so it will automatically use that environment, instead of " From 2e702d0887fb2597052158f094a1d4b57bf793d9 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 15:50:09 +0800 Subject: [PATCH 073/101] Clean up do_install --- pipenv/cli.py | 1 - pipenv/core.py | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 49722dae..01c7225b 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -407,7 +407,6 @@ def install( lock=lock, ignore_pipfile=ignore_pipfile, skip_lock=skip_lock, - verbose=(environments.PIPENV_VERBOSITY > 0), requirements=requirements, sequential=sequential, pre=pre, diff --git a/pipenv/core.py b/pipenv/core.py index 57db1325..352fe7a1 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1630,7 +1630,6 @@ def do_install( lock=True, ignore_pipfile=False, skip_lock=False, - verbose=False, requirements=False, sequential=False, pre=False, @@ -1832,7 +1831,7 @@ def do_install( ignore_pipfile=ignore_pipfile, system=system, skip_lock=skip_lock, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), concurrent=concurrent, deploy=deploy, pre=pre, @@ -1859,7 +1858,7 @@ def do_install( allow_global=system, selective_upgrade=selective_upgrade, no_deps=False, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), pre=pre, requirements_dir=requirements_directory.name, index=index, @@ -1929,7 +1928,7 @@ def do_install( system=system, allow_global=system, concurrent=concurrent, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), keep_outdated=keep_outdated, requirements_dir=requirements_directory, deploy=deploy, From d3987becabb66d832ad1f0196bb92ae1ccc034f1 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 15:51:05 +0800 Subject: [PATCH 074/101] Clean up do_uninstall --- pipenv/cli.py | 1 - pipenv/core.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 01c7225b..7559f70f 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -492,7 +492,6 @@ def uninstall( lock=lock, all_dev=all_dev, all=all, - verbose=(environments.PIPENV_VERBOSITY > 0), keep_outdated=keep_outdated, pypi_mirror=pypi_mirror, ) diff --git a/pipenv/core.py b/pipenv/core.py index 352fe7a1..dd4113a1 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1948,7 +1948,6 @@ def do_uninstall( lock=False, all_dev=False, all=False, - verbose=False, keep_outdated=False, pypi_mirror=None, ): @@ -1966,7 +1965,7 @@ def do_uninstall( click.echo( crayons.normal(u"Un-installing all packages from virtualenv…", bold=True) ) - do_purge(allow_global=system, verbose=verbose) + do_purge(allow_global=system, verbose=(environments.PIPENV_VERBOSITY > 0)) sys.exit(0) # Uninstall [dev-packages], if --dev was provided. if all_dev: @@ -1992,7 +1991,7 @@ def do_uninstall( cmd = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=system)), package_name ) - if verbose: + if environments.PIPENV_VERBOSITY > 0: click.echo("$ {0}".format(cmd)) c = delegator.run(cmd) click.echo(crayons.blue(c.out)) From 906c9cc874d1c3b36cb649183cf07da5234a7ea8 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 15:54:24 +0800 Subject: [PATCH 075/101] Clean up do_lock --- pipenv/cli.py | 5 +---- pipenv/core.py | 9 +++------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 7559f70f..017a7ffb 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -566,7 +566,6 @@ def lock( if requirements: do_init(dev=dev, requirements=requirements, pypi_mirror=pypi_mirror) do_lock( - verbose=(environments.PIPENV_VERBOSITY > 0), clear=clear, pre=pre, keep_outdated=keep_outdated, @@ -864,9 +863,7 @@ def update( ) sys.exit(1) - verbose = (environments.PIPENV_VERBOSITY > 0) do_lock( - verbose=verbose, clear=clear, pre=pre, keep_outdated=keep_outdated, @@ -880,7 +877,7 @@ def update( bare=bare, dont_upgrade=False, user=False, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), clear=clear, unused=False, sequential=sequential, diff --git a/pipenv/core.py b/pipenv/core.py index dd4113a1..4c489dbd 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -905,7 +905,6 @@ def get_downloads_info(names_map, section): def do_lock( - verbose=False, system=False, clear=False, pre=False, @@ -981,7 +980,7 @@ def do_lock( results = venv_resolve_deps( deps, which=which, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), project=project, clear=clear, pre=pre, @@ -1002,7 +1001,7 @@ def do_lock( project, pip_freeze, which=which, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), clear=clear, pre=pre, allow_global=system, @@ -1012,7 +1011,7 @@ def do_lock( vcs_results = venv_resolve_deps( vcs_lines, which=which, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), project=project, clear=clear, pre=pre, @@ -1194,7 +1193,6 @@ def do_init( system=system, pre=pre, keep_outdated=keep_outdated, - verbose=verbose, write=True, pypi_mirror=pypi_mirror, ) @@ -1222,7 +1220,6 @@ def do_init( system=system, pre=pre, keep_outdated=keep_outdated, - verbose=verbose, write=True, pypi_mirror=pypi_mirror, ) From 62e9a2e2e2a2240c8e91c18b4982e21300b17171 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 15:55:26 +0800 Subject: [PATCH 076/101] Clean up do_sync --- pipenv/cli.py | 2 -- pipenv/core.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 017a7ffb..5ba15cef 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -877,7 +877,6 @@ def update( bare=bare, dont_upgrade=False, user=False, - verbose=(environments.PIPENV_VERBOSITY > 0), clear=clear, unused=False, sequential=sequential, @@ -1012,7 +1011,6 @@ def sync( bare=bare, dont_upgrade=dont_upgrade, user=user, - verbose=(environments.PIPENV_VERBOSITY > 0), clear=clear, unused=unused, sequential=sequential, diff --git a/pipenv/core.py b/pipenv/core.py index 4c489dbd..c483cd0c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2410,7 +2410,6 @@ def do_sync( bare=False, dont_upgrade=False, user=False, - verbose=False, clear=False, unused=False, sequential=False, @@ -2441,7 +2440,7 @@ def do_sync( requirements_dir = TemporaryDirectory(suffix="-requirements", prefix="pipenv-") do_init( dev=dev, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), concurrent=(not sequential), requirements_dir=requirements_dir, ignore_pipfile=True, # Don't check if Pipfile and lock match. From 1fe1cc59b0a16f0f6d509ed20b1537d1439f6e1a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 15:56:06 +0800 Subject: [PATCH 077/101] Clean up do_clean --- pipenv/cli.py | 1 - pipenv/core.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 5ba15cef..d71325a7 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -1055,7 +1055,6 @@ def clean(ctx, three=None, python=None, dry_run=False, bare=False, user=False): ctx=ctx, three=three, python=python, dry_run=dry_run, - verbose=(environments.PIPENV_VERBOSITY > 0), ) diff --git a/pipenv/core.py b/pipenv/core.py index c483cd0c..e36ed5a0 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2458,7 +2458,6 @@ def do_clean( python=None, dry_run=False, bare=False, - verbose=False, pypi_mirror=None, ): # Ensure that virtualenv is available. @@ -2469,7 +2468,7 @@ def do_clean( # Remove known "bad packages" from the list. for bad_package in BAD_PACKAGES: if bad_package in installed_package_names: - if verbose: + if environments.PIPENV_VERBOSITY > 0: click.echo("Ignoring {0}.".format(repr(bad_package)), err=True) del installed_package_names[installed_package_names.index(bad_package)] # Intelligently detect if --dev should be used or not. From bcc0b7b3a7670b28bc0ac5ab471798ac1e618461 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 15:58:00 +0800 Subject: [PATCH 078/101] Clean up venv_resolve_deps --- pipenv/core.py | 2 -- pipenv/utils.py | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index e36ed5a0..d6eff14c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -980,7 +980,6 @@ def do_lock( results = venv_resolve_deps( deps, which=which, - verbose=(environments.PIPENV_VERBOSITY > 0), project=project, clear=clear, pre=pre, @@ -1011,7 +1010,6 @@ def do_lock( vcs_results = venv_resolve_deps( vcs_lines, which=which, - verbose=(environments.PIPENV_VERBOSITY > 0), project=project, clear=clear, pre=pre, diff --git a/pipenv/utils.py b/pipenv/utils.py index 286b1866..5d5cc195 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -41,6 +41,7 @@ except ImportError: from distutils.spawn import find_executable from contextlib import contextmanager +from . import environments from .pep508checker import lookup from .environments import PIPENV_MAX_ROUNDS, PIPENV_CACHE_DIR, PIPENV_MAX_RETRIES @@ -343,7 +344,6 @@ def venv_resolve_deps( which, project, pre=False, - verbose=False, clear=False, allow_global=False, pypi_mirror=None, @@ -354,6 +354,7 @@ def venv_resolve_deps( if not deps: return [] + verbose = (environments.PIPENV_VERBOSITY > 0) resolver = escape_grouped_arguments(resolver.__file__.rstrip("co")) cmd = "{0} {1} {2} {3} {4} {5}".format( escape_grouped_arguments(which("python", allow_global=allow_global)), @@ -375,7 +376,7 @@ def venv_resolve_deps( click_echo(c.out, err=True) click_echo(c.err, err=True) else: - click_echo(c.err[int(len(c.err) / 2) - 1 :], err=True) + click_echo(c.err[(int(len(c.err) / 2) - 1):], err=True) sys.exit(c.return_code) if verbose: click_echo(c.out.split("RESULTS:")[0], err=True) From 6ac323692b5a38e3f7d8a9db2ccd995ab1ccbe5c Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 15:58:53 +0800 Subject: [PATCH 079/101] Clean up get_vcs_deps --- pipenv/core.py | 1 - pipenv/utils.py | 1 - 2 files changed, 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index d6eff14c..39e2b342 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1000,7 +1000,6 @@ def do_lock( project, pip_freeze, which=which, - verbose=(environments.PIPENV_VERBOSITY > 0), clear=clear, pre=pre, allow_global=system, diff --git a/pipenv/utils.py b/pipenv/utils.py index 5d5cc195..4401228e 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1205,7 +1205,6 @@ def get_vcs_deps( project, pip_freeze=None, which=None, - verbose=False, clear=False, pre=False, allow_global=False, From 6f3a882c711d78c6fc3dd5c4b765cb5ec16111c3 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 15:59:32 +0800 Subject: [PATCH 080/101] Remove verbose variable --- pipenv/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 4401228e..987645e6 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -354,13 +354,12 @@ def venv_resolve_deps( if not deps: return [] - verbose = (environments.PIPENV_VERBOSITY > 0) resolver = escape_grouped_arguments(resolver.__file__.rstrip("co")) cmd = "{0} {1} {2} {3} {4} {5}".format( escape_grouped_arguments(which("python", allow_global=allow_global)), resolver, "--pre" if pre else "", - "--verbose" if verbose else "", + "--verbose" if (environments.PIPENV_VERBOSITY > 0) else "", "--clear" if clear else "", "--system" if allow_global else "", ) @@ -372,13 +371,13 @@ def venv_resolve_deps( try: assert c.return_code == 0 except AssertionError: - if verbose: + if environments.PIPENV_VERBOSITY > 0: click_echo(c.out, err=True) click_echo(c.err, err=True) else: click_echo(c.err[(int(len(c.err) / 2) - 1):], err=True) sys.exit(c.return_code) - if verbose: + if environments.PIPENV_VERBOSITY > 0: click_echo(c.out.split("RESULTS:")[0], err=True) try: return json.loads(c.out.split("RESULTS:")[1].strip()) From 30570478a85ced9a200445d23b425642219f8040 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 16:04:44 +0800 Subject: [PATCH 081/101] Clean up resolve_deps and actually_resolve_deps --- pipenv/resolver.py | 7 ++----- pipenv/utils.py | 12 +++--------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 72db9429..e4c9b09b 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -20,7 +20,6 @@ def which(*args, **kwargs): def main(): - is_verbose = "--verbose" in " ".join(sys.argv) do_pre = "--pre" in " ".join(sys.argv) do_clear = "--clear" in " ".join(sys.argv) is_debug = "--debug" in " ".join(sys.argv) @@ -36,7 +35,7 @@ def main(): os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) os.environ["PIP_PYTHON_PATH"] = sys.executable - if is_verbose: + if int(os.environ.get("PIPENV_VERBOSITY", 0)) > 0: logging.getLogger("notpip").setLevel(logging.INFO) if is_debug: # Shit's getting real at this point. @@ -56,7 +55,7 @@ def main(): else None ) - def resolve(packages, pre, project, sources, verbose, clear, system): + def resolve(packages, pre, project, sources, clear, system): return resolve_deps( packages, which, @@ -64,7 +63,6 @@ def main(): pre=pre, sources=sources, clear=clear, - verbose=verbose, allow_global=system, ) @@ -81,7 +79,6 @@ def main(): pre=do_pre, project=project, sources=sources, - verbose=is_verbose, clear=do_clear, system=system, ) diff --git a/pipenv/utils.py b/pipenv/utils.py index 987645e6..f82f38e4 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -220,14 +220,11 @@ def actually_resolve_deps( markers_lookup, project, sources, - verbose, clear, pre, req_dir=None, ): - from .vendor.packaging.markers import default_environment from .patched.notpip._internal import basecommand - from .patched.notpip._internal.cmdoptions import no_binary, only_binary from .patched.notpip._internal.req import parse_requirements from .patched.notpip._internal.exceptions import DistributionNotFound from .patched.notpip._vendor.requests.exceptions import HTTPError @@ -273,7 +270,7 @@ def actually_resolve_deps( pip_args = [] if sources: pip_args = prepare_pip_source_args(sources, pip_args) - if verbose: + if environments.PIPENV_VERBOSITY > 0: print("Using pip: {0}".format(" ".join(pip_args))) with NamedTemporaryFile( mode="w", @@ -296,7 +293,7 @@ def actually_resolve_deps( constraints_file, finder=pypi.finder, session=pypi.session, options=pip_options ) constraints = [c for c in constraints] - if verbose: + if environments.PIPENV_VERBOSITY > 0: logging.log.verbose = True piptools_logging.log.verbose = True resolved_tree = set() @@ -391,7 +388,6 @@ def resolve_deps( which, project, sources=None, - verbose=False, python=False, clear=False, pre=False, @@ -420,7 +416,6 @@ def resolve_deps( markers_lookup, project, sources, - verbose, clear, pre, req_dir=req_dir, @@ -443,7 +438,6 @@ def resolve_deps( markers_lookup, project, sources, - verbose, clear, pre, req_dir=req_dir, @@ -485,7 +479,7 @@ def resolve_deps( collected_hashes.append(release["digests"]["sha256"]) collected_hashes = ["sha256:" + s for s in collected_hashes] except (ValueError, KeyError, ConnectionError): - if verbose: + if environments.PIPENV_VERBOSITY > 0: click_echo( "{0}: Error generating hash for {1}".format( crayons.red("Warning", bold=True), name From 3de58e1276a1b24c0905f63a3969a0eb001e5f6a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 16:06:03 +0800 Subject: [PATCH 082/101] Clean up do_init --- pipenv/core.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 39e2b342..83121073 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1122,7 +1122,6 @@ def do_init( allow_global=False, ignore_pipfile=False, skip_lock=False, - verbose=False, system=False, concurrent=True, deploy=False, @@ -1225,7 +1224,7 @@ def do_init( requirements=requirements, allow_global=allow_global, skip_lock=skip_lock, - verbose=verbose, + verbose=environments.PIPENV_VERBOSITY > 0, concurrent=concurrent, requirements_dir=requirements_dir.name, pypi_mirror=pypi_mirror, @@ -1825,7 +1824,6 @@ def do_install( ignore_pipfile=ignore_pipfile, system=system, skip_lock=skip_lock, - verbose=(environments.PIPENV_VERBOSITY > 0), concurrent=concurrent, deploy=deploy, pre=pre, @@ -1922,7 +1920,6 @@ def do_install( system=system, allow_global=system, concurrent=concurrent, - verbose=(environments.PIPENV_VERBOSITY > 0), keep_outdated=keep_outdated, requirements_dir=requirements_directory, deploy=deploy, @@ -2437,7 +2434,6 @@ def do_sync( requirements_dir = TemporaryDirectory(suffix="-requirements", prefix="pipenv-") do_init( dev=dev, - verbose=(environments.PIPENV_VERBOSITY > 0), concurrent=(not sequential), requirements_dir=requirements_dir, ignore_pipfile=True, # Don't check if Pipfile and lock match. From 5770452095fd0f52a8d1ead6ad4ecfc0529fc957 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 16:07:11 +0800 Subject: [PATCH 083/101] Clean up do_install_dependencies --- pipenv/core.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 83121073..61d2be43 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -632,7 +632,6 @@ def do_install_dependencies( allow_global=False, ignore_hashes=False, skip_lock=False, - verbose=False, concurrent=True, requirements_dir=None, pypi_mirror=False, @@ -648,7 +647,7 @@ def do_install_dependencies( c.block() if "Ignoring" in c.out: click.echo(crayons.yellow(c.out.strip())) - elif verbose: + elif environments.PIPENV_VERBOSITY > 0: click.echo(crayons.blue(c.out or c.err)) # The Installation failed… if c.return_code != 0: @@ -731,7 +730,7 @@ def do_install_dependencies( ignore_hashes=ignore_hash, allow_global=allow_global, no_deps=no_deps, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), block=block, index=index, requirements_dir=requirements_dir, @@ -760,7 +759,7 @@ def do_install_dependencies( ignore_hashes=ignore_hash, allow_global=allow_global, no_deps=no_deps, - verbose=verbose, + verbose=(environments.PIPENV_VERBOSITY > 0), index=index, requirements_dir=requirements_dir, extra_indexes=extra_indexes, @@ -1224,7 +1223,6 @@ def do_init( requirements=requirements, allow_global=allow_global, skip_lock=skip_lock, - verbose=environments.PIPENV_VERBOSITY > 0, concurrent=concurrent, requirements_dir=requirements_dir.name, pypi_mirror=pypi_mirror, From 4894aa83d50e7d8832b48f987d1923d4cbc64e72 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 16:08:18 +0800 Subject: [PATCH 084/101] Clean up pip_install --- pipenv/core.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 61d2be43..7ed4de0c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -730,7 +730,6 @@ def do_install_dependencies( ignore_hashes=ignore_hash, allow_global=allow_global, no_deps=no_deps, - verbose=(environments.PIPENV_VERBOSITY > 0), block=block, index=index, requirements_dir=requirements_dir, @@ -759,7 +758,6 @@ def do_install_dependencies( ignore_hashes=ignore_hash, allow_global=allow_global, no_deps=no_deps, - verbose=(environments.PIPENV_VERBOSITY > 0), index=index, requirements_dir=requirements_dir, extra_indexes=extra_indexes, @@ -1247,7 +1245,6 @@ def pip_install( allow_global=False, ignore_hashes=False, no_deps=True, - verbose=False, block=True, index=None, pre=False, @@ -1260,7 +1257,7 @@ def pip_install( from notpip._vendor.pyparsing import ParseException from .vendor.requirementslib import Requirement - if verbose: + if environments.PIPENV_VERBOSITY > 0: click.echo( crayons.normal("Installing {0!r}".format(package_name), bold=True), err=True ) @@ -1354,13 +1351,13 @@ def pip_install( ), "sources": " ".join(prepare_pip_source_args(sources)), "src": src, - "verbose_flag": "--verbose" if verbose else "", + "verbose_flag": "--verbose" if environments.PIPENV_VERBOSITY > 0 else "", "install_reqs": install_reqs } pip_command = "{quoted_pip} install {pre} {src} {verbose_flag} {upgrade_strategy} {no_deps} {install_reqs} {sources}".format( **pip_args ) - if verbose: + if environments.PIPENV_VERBOSITY > 0: click.echo("$ {0}".format(pip_command), err=True) cache_dir = Path(PIPENV_CACHE_DIR) pip_config = { @@ -1848,7 +1845,6 @@ def do_install( allow_global=system, selective_upgrade=selective_upgrade, no_deps=False, - verbose=(environments.PIPENV_VERBOSITY > 0), pre=pre, requirements_dir=requirements_directory.name, index=index, From 6622bf15f2a342a9539a2ead59b387896a6e9a5a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 16:09:02 +0800 Subject: [PATCH 085/101] Clean up do_purge --- pipenv/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 7ed4de0c..1f73ed38 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1064,7 +1064,7 @@ def do_lock( return lockfile -def do_purge(bare=False, downloads=False, allow_global=False, verbose=False): +def do_purge(bare=False, downloads=False, allow_global=False): """Executes the purge functionality.""" from .vendor.requirementslib.models.requirements import Requirement @@ -1105,7 +1105,7 @@ def do_purge(bare=False, downloads=False, allow_global=False, verbose=False): escape_grouped_arguments(which_pip(allow_global=allow_global)), " ".join(actually_installed), ) - if verbose: + if environments.PIPENV_VERBOSITY > 0: click.echo("$ {0}".format(command)) c = delegator.run(command) if not bare: @@ -1950,7 +1950,7 @@ def do_uninstall( click.echo( crayons.normal(u"Un-installing all packages from virtualenv…", bold=True) ) - do_purge(allow_global=system, verbose=(environments.PIPENV_VERBOSITY > 0)) + do_purge(allow_global=system) sys.exit(0) # Uninstall [dev-packages], if --dev was provided. if all_dev: From b795d5bd72a19dad700714d4e7a0d72eab0d2300 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 25 Jul 2018 17:25:19 +0800 Subject: [PATCH 086/101] Add wrapper function to PIPENV_VERBOSITY --- pipenv/core.py | 18 +++++++++--------- pipenv/environments.py | 8 ++++++++ pipenv/utils.py | 12 ++++++------ 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 1f73ed38..db60ee94 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -647,7 +647,7 @@ def do_install_dependencies( c.block() if "Ignoring" in c.out: click.echo(crayons.yellow(c.out.strip())) - elif environments.PIPENV_VERBOSITY > 0: + elif environments.is_verbose(): click.echo(crayons.blue(c.out or c.err)) # The Installation failed… if c.return_code != 0: @@ -1105,7 +1105,7 @@ def do_purge(bare=False, downloads=False, allow_global=False): escape_grouped_arguments(which_pip(allow_global=allow_global)), " ".join(actually_installed), ) - if environments.PIPENV_VERBOSITY > 0: + if environments.is_verbose(): click.echo("$ {0}".format(command)) c = delegator.run(command) if not bare: @@ -1257,7 +1257,7 @@ def pip_install( from notpip._vendor.pyparsing import ParseException from .vendor.requirementslib import Requirement - if environments.PIPENV_VERBOSITY > 0: + if environments.is_verbose(): click.echo( crayons.normal("Installing {0!r}".format(package_name), bold=True), err=True ) @@ -1351,13 +1351,13 @@ def pip_install( ), "sources": " ".join(prepare_pip_source_args(sources)), "src": src, - "verbose_flag": "--verbose" if environments.PIPENV_VERBOSITY > 0 else "", + "verbose_flag": "--verbose" if environments.is_verbose() else "", "install_reqs": install_reqs } pip_command = "{quoted_pip} install {pre} {src} {verbose_flag} {upgrade_strategy} {no_deps} {install_reqs} {sources}".format( **pip_args ) - if environments.PIPENV_VERBOSITY > 0: + if environments.is_verbose(): click.echo("$ {0}".format(pip_command), err=True) cache_dir = Path(PIPENV_CACHE_DIR) pip_config = { @@ -1526,8 +1526,8 @@ def format_pip_output(out, r=None): def warn_in_virtualenv(): # Only warn if pipenv isn't already active. pipenv_active = os.environ.get("PIPENV_ACTIVE") - if ((environments.PIPENV_USE_SYSTEM or environments.PIPENV_VIRTUALENV) - and not (pipenv_active or environments.PIPENV_VERBOSITY < 0)): + if ((environments.PIPENV_USE_SYSTEM or environments.PIPENV_VIRTUALENV) and + not (pipenv_active or environments.is_quiet())): click.echo( "{0}: Pipenv found itself running within a virtual environment, " "so it will automatically use that environment, instead of " @@ -1976,7 +1976,7 @@ def do_uninstall( cmd = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=system)), package_name ) - if environments.PIPENV_VERBOSITY > 0: + if environments.is_verbose(): click.echo("$ {0}".format(cmd)) c = delegator.run(cmd) click.echo(crayons.blue(c.out)) @@ -2455,7 +2455,7 @@ def do_clean( # Remove known "bad packages" from the list. for bad_package in BAD_PACKAGES: if bad_package in installed_package_names: - if environments.PIPENV_VERBOSITY > 0: + if environments.is_verbose(): click.echo("Ignoring {0}.".format(repr(bad_package)), err=True) del installed_package_names[installed_package_names.index(bad_package)] # Intelligently detect if --dev should be used or not. diff --git a/pipenv/environments.py b/pipenv/environments.py index e3fd793e..e28400ac 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -189,6 +189,14 @@ if interactive. """ +def is_verbose(threshold=1): + return PIPENV_VERBOSITY >= threshold + + +def is_quiet(threshold=-1): + return PIPENV_VERBOSITY <= threshold + + # Internal, support running in a different Python from sys.executable. PIPENV_PYTHON = os.environ.get("PIPENV_PYTHON") diff --git a/pipenv/utils.py b/pipenv/utils.py index f82f38e4..9cf62366 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -270,7 +270,7 @@ def actually_resolve_deps( pip_args = [] if sources: pip_args = prepare_pip_source_args(sources, pip_args) - if environments.PIPENV_VERBOSITY > 0: + if environments.is_verbose(): print("Using pip: {0}".format(" ".join(pip_args))) with NamedTemporaryFile( mode="w", @@ -293,7 +293,7 @@ def actually_resolve_deps( constraints_file, finder=pypi.finder, session=pypi.session, options=pip_options ) constraints = [c for c in constraints] - if environments.PIPENV_VERBOSITY > 0: + if environments.is_verbose(): logging.log.verbose = True piptools_logging.log.verbose = True resolved_tree = set() @@ -356,7 +356,7 @@ def venv_resolve_deps( escape_grouped_arguments(which("python", allow_global=allow_global)), resolver, "--pre" if pre else "", - "--verbose" if (environments.PIPENV_VERBOSITY > 0) else "", + "--verbose" if (environments.is_verbose()) else "", "--clear" if clear else "", "--system" if allow_global else "", ) @@ -368,13 +368,13 @@ def venv_resolve_deps( try: assert c.return_code == 0 except AssertionError: - if environments.PIPENV_VERBOSITY > 0: + if environments.is_verbose(): click_echo(c.out, err=True) click_echo(c.err, err=True) else: click_echo(c.err[(int(len(c.err) / 2) - 1):], err=True) sys.exit(c.return_code) - if environments.PIPENV_VERBOSITY > 0: + if environments.is_verbose(): click_echo(c.out.split("RESULTS:")[0], err=True) try: return json.loads(c.out.split("RESULTS:")[1].strip()) @@ -479,7 +479,7 @@ def resolve_deps( collected_hashes.append(release["digests"]["sha256"]) collected_hashes = ["sha256:" + s for s in collected_hashes] except (ValueError, KeyError, ConnectionError): - if environments.PIPENV_VERBOSITY > 0: + if environments.is_verbose(): click_echo( "{0}: Error generating hash for {1}".format( crayons.red("Warning", bold=True), name From 5c395f1c3b43d4715d5ebf84fe201a564f15d28e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 26 Jul 2018 13:53:57 +0800 Subject: [PATCH 087/101] Add PIPENV_QUIET and PIPENV_VERBOSE env vars This enables flag-based verbosity control from environment variables, similar to command line flags. --- pipenv/environments.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/pipenv/environments.py b/pipenv/environments.py index e28400ac..b589b9ef 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -146,6 +146,13 @@ Default is to not mirror PyPI, i.e. use the real one, pypi.org. The ``--pypi-mirror`` command line flag overwrites this. """ +PIPENV_QUIET = bool(os.environ.get("PIPENV_QUIET")) +"""If set, makes Pipenv quieter. + +Default is unset, for normal verbosity. ``PIPENV_VERBOSE`` overrides this. +See also ``PIPENV_VERBOSITY``. +""" + PIPENV_SHELL = os.environ.get("PIPENV_SHELL") """An absolute path to the preferred shell for ``pipenv shell``. @@ -174,12 +181,32 @@ PIPENV_VENV_IN_PROJECT = bool(os.environ.get("PIPENV_VENV_IN_PROJECT")) Default is to create new virtual environments in a global location. """ -PIPENV_VERBOSITY = int(os.environ.get("PIPENV_VERBOSITY", 0)) +PIPENV_VERBOSE = bool(os.environ.get("PIPENV_VERBOSE")) +"""If set, makes Pipenv more wordy. + +Default is unset, for normal verbosity. This takes precedence over +``PIPENV_QUIET``. See also ``PIPENV_VERBOSITY``. +""" + +PIPENV_VERBOSITY = os.environ.get("PIPENV_VERBOSITY", "") """Verbosity setting for pipenv. Higher values make pipenv more verbose, lower values less so. Default is 0, -for normal verbosity. +for normal verbosity. This takes precedence over both ``PIPENV_QUIET`` and +``PIPENV_VERBOSE``. """ +# Consolidate the verbosity flags. +if PIPENV_VERBOSITY.isdigit(): + PIPENV_VERBOSITY = int(PIPENV_VERBOSITY) +else: + if PIPENV_VERBOSE: + PIPENV_VERBOSITY = 1 + elif PIPENV_QUIET: + PIPENV_VERBOSITY = -1 + else: + PIPENV_VERBOSITY = 0 +del PIPENV_QUIET +del PIPENV_VERBOSE PIPENV_YES = bool(os.environ.get("PIPENV_YES")) """If set, Pipenv automatically assumes "yes" at all prompts. From fece72b205a4d793690f031aa0804e949a481efd Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 26 Jul 2018 15:46:36 +0800 Subject: [PATCH 088/101] Make PIPENV_VERBOSITY undocumented --- news/2527.feature | 4 +-- pipenv/environments.py | 55 +++++++++++++++++++----------------------- 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/news/2527.feature b/news/2527.feature index e8148a59..ad8e5c30 100644 --- a/news/2527.feature +++ b/news/2527.feature @@ -1,2 +1,2 @@ -Added environment variable `PIPENV_VERBOSITY` to control output verbosity -without needing to pass options. +Added environment variables `PIPENV_VERBOSE` and `PIPENV_QUIET` to control +output verbosity without needing to pass options. diff --git a/pipenv/environments.py b/pipenv/environments.py index b589b9ef..0e21ca3b 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -150,7 +150,6 @@ PIPENV_QUIET = bool(os.environ.get("PIPENV_QUIET")) """If set, makes Pipenv quieter. Default is unset, for normal verbosity. ``PIPENV_VERBOSE`` overrides this. -See also ``PIPENV_VERBOSITY``. """ PIPENV_SHELL = os.environ.get("PIPENV_SHELL") @@ -185,29 +184,9 @@ PIPENV_VERBOSE = bool(os.environ.get("PIPENV_VERBOSE")) """If set, makes Pipenv more wordy. Default is unset, for normal verbosity. This takes precedence over -``PIPENV_QUIET``. See also ``PIPENV_VERBOSITY``. +``PIPENV_QUIET``. """ -PIPENV_VERBOSITY = os.environ.get("PIPENV_VERBOSITY", "") -"""Verbosity setting for pipenv. - -Higher values make pipenv more verbose, lower values less so. Default is 0, -for normal verbosity. This takes precedence over both ``PIPENV_QUIET`` and -``PIPENV_VERBOSE``. -""" -# Consolidate the verbosity flags. -if PIPENV_VERBOSITY.isdigit(): - PIPENV_VERBOSITY = int(PIPENV_VERBOSITY) -else: - if PIPENV_VERBOSE: - PIPENV_VERBOSITY = 1 - elif PIPENV_QUIET: - PIPENV_VERBOSITY = -1 - else: - PIPENV_VERBOSITY = 0 -del PIPENV_QUIET -del PIPENV_VERBOSE - PIPENV_YES = bool(os.environ.get("PIPENV_YES")) """If set, Pipenv automatically assumes "yes" at all prompts. @@ -216,14 +195,6 @@ if interactive. """ -def is_verbose(threshold=1): - return PIPENV_VERBOSITY >= threshold - - -def is_quiet(threshold=-1): - return PIPENV_VERBOSITY <= threshold - - # Internal, support running in a different Python from sys.executable. PIPENV_PYTHON = os.environ.get("PIPENV_PYTHON") @@ -250,3 +221,27 @@ PIPENV_SHELL = ( # Internal, to tell whether the command line session is interactive. SESSION_IS_INTERACTIVE = bool(os.isatty(sys.stdout.fileno())) + + +# Internal, consolidated verbosity representation as an integer. The default +# level is 0, increased for wordiness and decreased for terseness. +PIPENV_VERBOSITY = os.environ.get("PIPENV_VERBOSITY", "") +if PIPENV_VERBOSITY.isdigit(): + PIPENV_VERBOSITY = int(PIPENV_VERBOSITY) +else: + if PIPENV_VERBOSE: + PIPENV_VERBOSITY = 1 + elif PIPENV_QUIET: + PIPENV_VERBOSITY = -1 + else: + PIPENV_VERBOSITY = 0 +del PIPENV_QUIET +del PIPENV_VERBOSE + + +def is_verbose(threshold=1): + return PIPENV_VERBOSITY >= threshold + + +def is_quiet(threshold=-1): + return PIPENV_VERBOSITY <= threshold From 236a6c4522128b51e456ff1ac1062dd5c490fc8a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 30 Jul 2018 15:37:10 +0800 Subject: [PATCH 089/101] Ignore OSError in is_virtual_environment check This works around a faulty virtual environment on my machine that makes os.access throw "OSError: too many level of symlinks". If a virtual environment is faulty, we can just ignore it. --- pipenv/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 9cf62366..d07d36ab 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1350,8 +1350,12 @@ def is_virtual_environment(path): if not path.is_dir(): return False for bindir_name in ('bin', 'Scripts'): - for python_like in path.joinpath(bindir_name).glob('python*'): - if python_like.is_file() and os.access(str(python_like), os.X_OK): + for python in path.joinpath(bindir_name).glob('python*'): + try: + exeness = python.is_file() and os.access(str(python), os.X_OK) + except OSError: + exeness = False + if exeness: return True return False From a89acdc589a19ddab730cff40a9f62768aaa6476 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 30 Jul 2018 15:41:25 +0800 Subject: [PATCH 090/101] News --- news/2676.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/2676.bugfix diff --git a/news/2676.bugfix b/news/2676.bugfix new file mode 100644 index 00000000..fb47d64b --- /dev/null +++ b/news/2676.bugfix @@ -0,0 +1 @@ +Prevent crashing when a virtual environment in ``WORKON_HOME`` is faulty. From 25370cae954310d067e6326f7d195f9c148379b4 Mon Sep 17 00:00:00 2001 From: Brett Randall Date: Wed, 25 Jul 2018 15:53:15 +1000 Subject: [PATCH 091/101] Fixed http->https on content from ghbtns.com, fixes mixed-content warning. --- docs/_templates/sidebarlogo.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index d348f5f0..5107e4de 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -17,7 +17,7 @@

Stay Informed

Receive updates on new releases and upcoming projects.

-

From 4cabc0fbab76a8c014132ee00de37d077ca342c6 Mon Sep 17 00:00:00 2001 From: Brett Randall Date: Wed, 25 Jul 2018 15:56:16 +1000 Subject: [PATCH 092/101] Added konami.js and switched to an alive target site. --- docs/_static/konami.js | 151 +++++++++++++++++++++++++++++++++++++ docs/_templates/hacks.html | 2 +- 2 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 docs/_static/konami.js diff --git a/docs/_static/konami.js b/docs/_static/konami.js new file mode 100644 index 00000000..e0028445 --- /dev/null +++ b/docs/_static/konami.js @@ -0,0 +1,151 @@ +/* + * Konami-JS ~ + * :: Now with support for touch events and multiple instances for + * :: those situations that call for multiple easter eggs! + * Code: https://github.com/snaptortoise/konami-js + * Copyright (c) 2009 George Mandis (georgemandis.com, snaptortoise.com) + * Version: 1.6.2 (7/17/2018) + * Licensed under the MIT License (http://opensource.org/licenses/MIT) + * Tested in: Safari 4+, Google Chrome 4+, Firefox 3+, IE7+, Mobile Safari 2.2.1+ and Android + */ + +var Konami = function (callback) { + var konami = { + addEvent: function (obj, type, fn, ref_obj) { + if (obj.addEventListener) + obj.addEventListener(type, fn, false); + else if (obj.attachEvent) { + // IE + obj["e" + type + fn] = fn; + obj[type + fn] = function () { + obj["e" + type + fn](window.event, ref_obj); + } + obj.attachEvent("on" + type, obj[type + fn]); + } + }, + removeEvent: function (obj, eventName, eventCallback) { + if (obj.removeEventListener) { + obj.removeEventListener(eventName, eventCallback); + } else if (obj.attachEvent) { + obj.detachEvent(eventName); + } + }, + input: "", + pattern: "38384040373937396665", + keydownHandler: function (e, ref_obj) { + if (ref_obj) { + konami = ref_obj; + } // IE + konami.input += e ? e.keyCode : event.keyCode; + if (konami.input.length > konami.pattern.length) { + konami.input = konami.input.substr((konami.input.length - konami.pattern.length)); + } + if (konami.input === konami.pattern) { + konami.code(konami._currentLink); + konami.input = ''; + e.preventDefault(); + return false; + } + }, + load: function (link) { + this._currentLink = link; + this.addEvent(document, "keydown", this.keydownHandler, this); + this.iphone.load(link); + }, + unload: function () { + this.removeEvent(document, 'keydown', this.keydownHandler); + this.iphone.unload(); + }, + code: function (link) { + window.location = link + }, + iphone: { + start_x: 0, + start_y: 0, + stop_x: 0, + stop_y: 0, + tap: false, + capture: false, + orig_keys: "", + keys: ["UP", "UP", "DOWN", "DOWN", "LEFT", "RIGHT", "LEFT", "RIGHT", "TAP", "TAP"], + input: [], + code: function (link) { + konami.code(link); + }, + touchmoveHandler: function (e) { + if (e.touches.length === 1 && konami.iphone.capture === true) { + var touch = e.touches[0]; + konami.iphone.stop_x = touch.pageX; + konami.iphone.stop_y = touch.pageY; + konami.iphone.tap = false; + konami.iphone.capture = false; + konami.iphone.check_direction(); + } + }, + touchendHandler: function () { + konami.iphone.input.push(konami.iphone.check_direction()); + + if (konami.iphone.input.length > konami.iphone.keys.length) konami.iphone.input.shift(); + + if (konami.iphone.input.length === konami.iphone.keys.length) { + var match = true; + for (var i = 0; i < konami.iphone.keys.length; i++) { + if (konami.iphone.input[i] !== konami.iphone.keys[i]) { + match = false; + } + } + if (match) { + konami.iphone.code(konami._currentLink); + } + } + }, + touchstartHandler: function (e) { + konami.iphone.start_x = e.changedTouches[0].pageX; + konami.iphone.start_y = e.changedTouches[0].pageY; + konami.iphone.tap = true; + konami.iphone.capture = true; + }, + load: function (link) { + this.orig_keys = this.keys; + konami.addEvent(document, "touchmove", this.touchmoveHandler); + konami.addEvent(document, "touchend", this.touchendHandler, false); + konami.addEvent(document, "touchstart", this.touchstartHandler); + }, + unload: function () { + konami.removeEvent(document, 'touchmove', this.touchmoveHandler); + konami.removeEvent(document, 'touchend', this.touchendHandler); + konami.removeEvent(document, 'touchstart', this.touchstartHandler); + }, + check_direction: function () { + x_magnitude = Math.abs(this.start_x - this.stop_x); + y_magnitude = Math.abs(this.start_y - this.stop_y); + x = ((this.start_x - this.stop_x) < 0) ? "RIGHT" : "LEFT"; + y = ((this.start_y - this.stop_y) < 0) ? "DOWN" : "UP"; + result = (x_magnitude > y_magnitude) ? x : y; + result = (this.tap === true) ? "TAP" : result; + return result; + } + } + } + + typeof callback === "string" && konami.load(callback); + if (typeof callback === "function") { + konami.code = callback; + konami.load(); + } + + return konami; +}; + + +if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') { + module.exports = Konami; +} else { + if (typeof define === 'function' && define.amd) { + define([], function() { + return Konami; + }); + } else { + window.Konami = Konami; + } +} diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html index f050901e..21348b04 100644 --- a/docs/_templates/hacks.html +++ b/docs/_templates/hacks.html @@ -49,7 +49,7 @@