From f3e4e73cff7ac91fc13da87f367b2bdbfb2b9d31 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 18:31:55 -0400 Subject: [PATCH] Fix resolution using `dependency_links` with ssh - Exclude VCS SSH uris from hashing - Add additional resilience to the piptools resolver - Fixes #2613 Signed-off-by: Dan Ryan --- news/2643.bugfix | 1 + pipenv/patched/piptools/repositories/pypi.py | 28 +++++---- .../vendoring/patches/patched/piptools.patch | 57 ++++++++++++------- 3 files changed, 55 insertions(+), 31 deletions(-) create mode 100644 news/2643.bugfix diff --git a/news/2643.bugfix b/news/2643.bugfix new file mode 100644 index 00000000..566879af --- /dev/null +++ b/news/2643.bugfix @@ -0,0 +1 @@ +Dependency links to private repositories defined via ``ssh://`` schemes will now install correctly and skip hashing as long as ``PIP_PROCESS_DEPENDENCY_LINKS=1``. diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 2f746094..feab21b5 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -21,18 +21,16 @@ from .._compat import ( SafeFileCache, ) -from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement -from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier -from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable -from pipenv.patched.notpip._vendor.pyparsing import ParseException +from pipenv.patched.notpip._vendor.packaging.requirements import Requirement +from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, Specifier +from pipenv.patched.notpip._vendor.packaging.markers import Op, Value, Variable from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.vcs import VcsSupport -from ..cache import CACHE_DIR from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound -from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, - make_install_requirement, format_requirement, dedup, clean_requires_python) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, + make_install_requirement, clean_requires_python) from .base import BaseRepository @@ -64,9 +62,10 @@ class HashCache(SafeFileCache): def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') + vcs = VcsSupport() + orig_scheme = location.scheme new_location = copy.deepcopy(location) - if any(new_location.url.startswith(vcs) for vcs in vcs_uris): + if orig_scheme in vcs.all_schemes: new_location.url = new_location.url.split("+", 1)[-1] can_hash = new_location.hash if can_hash: @@ -280,6 +279,11 @@ class PyPIRepository(BaseRepository): setup_requires = {} dist = None if ireq.editable: + try: + from setuptools.build_meta import _run_setup + _run_setup(ireq.setup_py) + except (ImportError, InstallationError): + pass try: dist = ireq.get_dist() except InstallationError: @@ -429,6 +433,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() + vcs = VcsSupport() + if ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + return set() + if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 56696e08..b527470b 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..245e9ce 100644 +index 1c4b943..9461709 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,9 +1,10 @@ @@ -34,7 +34,7 @@ index 1c4b943..245e9ce 100644 from contextlib import contextmanager from shutil import rmtree -@@ -15,13 +16,24 @@ from .._compat import ( +@@ -15,13 +16,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -44,25 +44,23 @@ index 1c4b943..245e9ce 100644 + SafeFileCache, ) -+from pip._vendor.packaging.requirements import InvalidRequirement, Requirement -+from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -+from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier -+from pip._vendor.packaging.markers import Marker, Op, Value, Variable -+from pip._vendor.pyparsing import ParseException +-from ..cache import CACHE_DIR ++from pip._vendor.packaging.requirements import Requirement ++from pip._vendor.packaging.specifiers import SpecifierSet, Specifier ++from pip._vendor.packaging.markers import Op, Value, Variable +from pip._internal.exceptions import InstallationError ++from pip._internal.vcs import VcsSupport + - from ..cache import CACHE_DIR +from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound --from ..utils import (fs_str, is_pinned_requirement, lookup_table, + from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) -+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, -+ make_install_requirement, format_requirement, dedup, clean_requires_python) ++ make_install_requirement, clean_requires_python) + from .base import BaseRepository -@@ -37,6 +49,44 @@ except ImportError: +@@ -37,6 +47,45 @@ except ImportError: from pip.wheel import WheelCache @@ -81,9 +79,10 @@ index 1c4b943..245e9ce 100644 + def get_hash(self, location): + # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it + hash_value = None -+ vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') ++ vcs = VcsSupport() ++ orig_scheme = location.scheme + new_location = copy.deepcopy(location) -+ if any(new_location.url.startswith(vcs) for vcs in vcs_uris): ++ if orig_scheme in vcs.all_schemes: + new_location.url = new_location.url.split("+", 1)[-1] + can_hash = new_location.hash + if can_hash: @@ -107,7 +106,7 @@ index 1c4b943..245e9ce 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,10 +96,11 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +95,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -121,7 +120,7 @@ index 1c4b943..245e9ce 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -74,11 +125,15 @@ class PyPIRepository(BaseRepository): +@@ -74,11 +124,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -139,7 +138,7 @@ index 1c4b943..245e9ce 100644 def freshen_build_caches(self): """ -@@ -114,10 +169,14 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +168,14 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -156,7 +155,7 @@ index 1c4b943..245e9ce 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +185,71 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +184,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -231,7 +230,7 @@ index 1c4b943..245e9ce 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,20 +274,40 @@ class PyPIRepository(BaseRepository): +@@ -155,20 +273,45 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -243,6 +242,11 @@ index 1c4b943..245e9ce 100644 + dist = None + if ireq.editable: + try: ++ from setuptools.build_meta import _run_setup ++ _run_setup(ireq.setup_py) ++ except (ImportError, InstallationError): ++ pass ++ try: + dist = ireq.get_dist() + except InstallationError: + ireq.run_egg_info() @@ -276,7 +280,7 @@ index 1c4b943..245e9ce 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +327,97 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +331,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -377,7 +381,18 @@ index 1c4b943..245e9ce 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -217,24 +436,22 @@ class PyPIRepository(BaseRepository): +@@ -210,6 +433,10 @@ class PyPIRepository(BaseRepository): + if ireq.editable: + return set() + ++ vcs = VcsSupport() ++ if ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: ++ return set() ++ + if not is_pinned_requirement(ireq): + raise TypeError( + "Expected pinned requirement, got {}".format(ireq)) +@@ -217,24 +444,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint.