diff --git a/news/4637.bugfix.rst b/news/4637.bugfix.rst new file mode 100644 index 00000000..a51906bd --- /dev/null +++ b/news/4637.bugfix.rst @@ -0,0 +1 @@ +Patched our vendored Pip to fix: Pipenv Lock (Or Install) Does Not Respect Index Specified For A Package. diff --git a/pipenv/core.py b/pipenv/core.py index 8f42cdb3..064656eb 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1418,7 +1418,11 @@ def pip_install( if not index and requirement.index: index = requirement.index if index and not extra_indexes: - extra_indexes = list(project.sources) + extra_indexes = [] + if requirement.index: + extra_indexes = list(filter(lambda d: d['name'] == requirement.index, project.sources)) + if not extra_indexes: + extra_indexes = list(project.sources) if requirement and requirement.vcs or requirement.editable: requirement.index = None # Install dependencies when a package is a non-editable VCS dependency. @@ -1438,6 +1442,8 @@ def pip_install( project, index, extra_indexes=extra_indexes, trusted_hosts=trusted_hosts, pypi_mirror=pypi_mirror ) + if requirement.index in sources: + sources = list(filter(lambda d: d.get('name') == requirement.index, sources)) if r: with open(r, "r") as fh: if "--hash" not in fh.read(): diff --git a/pipenv/patched/notpip/_internal/index/collector.py b/pipenv/patched/notpip/_internal/index/collector.py index 98ac9d2e..fc532c40 100644 --- a/pipenv/patched/notpip/_internal/index/collector.py +++ b/pipenv/patched/notpip/_internal/index/collector.py @@ -17,6 +17,7 @@ from optparse import Values from typing import ( Callable, Iterable, + Dict, List, MutableMapping, NamedTuple, @@ -445,15 +446,18 @@ class LinkCollector: self, session: PipSession, search_scope: SearchScope, + index_lookup: Optional[Dict[str, List[str]]] = None, ) -> None: self.search_scope = search_scope self.session = session + self.index_lookup = index_lookup if index_lookup else {} @classmethod def create( cls, session: PipSession, options: Values, - suppress_no_index: bool = False + suppress_no_index: bool = False, + index_lookup: Optional[Dict[str, List[str]]] = None, ) -> "LinkCollector": """ :param session: The Session to use to make requests. @@ -472,10 +476,10 @@ class LinkCollector: find_links = options.find_links or [] search_scope = SearchScope.create( - find_links=find_links, index_urls=index_urls, + find_links=find_links, index_urls=index_urls, index_lookup=index_lookup ) link_collector = LinkCollector( - session=session, search_scope=search_scope, + session=session, search_scope=search_scope, index_lookup=index_lookup ) return link_collector diff --git a/pipenv/patched/notpip/_internal/models/search_scope.py b/pipenv/patched/notpip/_internal/models/search_scope.py index 4b700407..58154c31 100644 --- a/pipenv/patched/notpip/_internal/models/search_scope.py +++ b/pipenv/patched/notpip/_internal/models/search_scope.py @@ -3,7 +3,7 @@ import logging import os import posixpath import urllib.parse -from typing import List +from typing import Dict, List, Optional from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name @@ -20,13 +20,14 @@ class SearchScope: Encapsulates the locations that pip is configured to search. """ - __slots__ = ["find_links", "index_urls"] + __slots__ = ["find_links", "index_urls", "index_lookup"] @classmethod def create( cls, find_links: List[str], index_urls: List[str], + index_lookup: Optional[Dict[str, List[str]]] = None, ) -> "SearchScope": """ Create a SearchScope object after normalizing the `find_links`. @@ -60,15 +61,18 @@ class SearchScope: return cls( find_links=built_find_links, index_urls=index_urls, + index_lookup=index_lookup ) def __init__( self, find_links: List[str], index_urls: List[str], + index_lookup: Optional[Dict[str, List[str]]] = None, ) -> None: self.find_links = find_links self.index_urls = index_urls + self.index_lookup = index_lookup if index_lookup else {} def get_formatted_locations(self) -> str: lines = [] @@ -123,4 +127,7 @@ class SearchScope: loc = loc + '/' return loc - return [mkurl_pypi_url(url) for url in self.index_urls] + index_urls = self.index_urls + if project_name in self.index_lookup: + index_urls = [self.index_lookup[project_name]] + return [mkurl_pypi_url(url) for url in index_urls] diff --git a/pipenv/utils.py b/pipenv/utils.py index d43158bb..2abc0ccb 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -496,6 +496,15 @@ class Resolver: # directories into the initial constraint pool to be resolved with the # rest of the dependencies, while adding the files/vcs deps/paths themselves # to the lockfile directly + use_sources = None + if req.name in index_lookup: + use_sources = list(filter(lambda s: s.get('name') == index_lookup[req.name], sources)) + if not use_sources: + use_sources = sources + transient_resolver = cls( + [], req_dir, project, use_sources, index_lookup=index_lookup, + markers_lookup=markers_lookup, clear=clear, pre=pre + ) constraint_update, lockfile_update = cls.get_deps_from_req( req, resolver=transient_resolver, resolve_vcs=project.s.PIPENV_RESOLVE_VCS ) @@ -796,6 +805,16 @@ class Resolver: options=self.pip_options, session=self.session ) + index_mapping = {} + for source in self.sources: + if source.get('name'): + index_mapping[source['name']] = source['url'] + alt_index_lookup = {} + for req_name, index in self.index_lookup.items(): + if index_mapping.get(index): + alt_index_lookup[req_name] = index_mapping[index] + self._finder._link_collector.index_lookup = alt_index_lookup + self._finder._link_collector.search_scope.index_lookup = alt_index_lookup return self._finder @property @@ -818,10 +837,12 @@ class Resolver: def parsed_constraints(self): from pipenv.vendor.pip_shims import shims + pip_options = self.pip_options + pip_options.extra_index_urls = [] if self._parsed_constraints is None: self._parsed_constraints = shims.parse_requirements( self.constraint_file, finder=self.finder, session=self.session, - options=self.pip_options + options=pip_options ) return self._parsed_constraints @@ -876,6 +897,7 @@ class Resolver: from pipenv.vendor.pip_shims.shims import InstallationError from pipenv.exceptions import ResolutionFailure + self.constraints # For some reason its important to evaluate constraints before resolver context with temp_environ(), self.get_resolver() as resolver: try: results = resolver.resolve(self.constraints, check_supported_wheels=False) diff --git a/tasks/vendoring/patches/patched/pip_index_safety.patch b/tasks/vendoring/patches/patched/pip_index_safety.patch new file mode 100644 index 00000000..91b2235b --- /dev/null +++ b/tasks/vendoring/patches/patched/pip_index_safety.patch @@ -0,0 +1,102 @@ +diff --git a/pipenv/patched/pip/_internal/index/collector.py b/pipenv/patched/pip/_internal/index/collector.py +index 98ac9d2e..fc532c40 100644 +--- a/pipenv/patched/pip/_internal/index/collector.py ++++ b/pipenv/patched/pip/_internal/index/collector.py +@@ -17,6 +17,7 @@ from optparse import Values + from typing import ( + Callable, + Iterable, ++ Dict, + List, + MutableMapping, + NamedTuple, +@@ -445,15 +446,18 @@ class LinkCollector: + self, + session: PipSession, + search_scope: SearchScope, ++ index_lookup: Optional[Dict[str, List[str]]] = None, + ) -> None: + self.search_scope = search_scope + self.session = session ++ self.index_lookup = index_lookup if index_lookup else {} + + @classmethod + def create( + cls, session: PipSession, + options: Values, +- suppress_no_index: bool = False ++ suppress_no_index: bool = False, ++ index_lookup: Optional[Dict[str, List[str]]] = None, + ) -> "LinkCollector": + """ + :param session: The Session to use to make requests. +@@ -472,10 +476,10 @@ class LinkCollector: + find_links = options.find_links or [] + + search_scope = SearchScope.create( +- find_links=find_links, index_urls=index_urls, ++ find_links=find_links, index_urls=index_urls, index_lookup=index_lookup + ) + link_collector = LinkCollector( +- session=session, search_scope=search_scope, ++ session=session, search_scope=search_scope, index_lookup=index_lookup + ) + return link_collector + +diff --git a/pipenv/patched/pip/_internal/models/search_scope.py b/pipenv/patched/pip/_internal/models/search_scope.py +index 4b700407..58154c31 100644 +--- a/pipenv/patched/pip/_internal/models/search_scope.py ++++ b/pipenv/patched/pip/_internal/models/search_scope.py +@@ -3,7 +3,7 @@ import logging + import os + import posixpath + import urllib.parse +-from typing import List ++from typing import Dict, List, Optional + + from pip._vendor.packaging.utils import canonicalize_name + +@@ -20,13 +20,14 @@ class SearchScope: + Encapsulates the locations that pip is configured to search. + """ + +- __slots__ = ["find_links", "index_urls"] ++ __slots__ = ["find_links", "index_urls", "index_lookup"] + + @classmethod + def create( + cls, + find_links: List[str], + index_urls: List[str], ++ index_lookup: Optional[Dict[str, List[str]]] = None, + ) -> "SearchScope": + """ + Create a SearchScope object after normalizing the `find_links`. +@@ -60,15 +61,18 @@ class SearchScope: + return cls( + find_links=built_find_links, + index_urls=index_urls, ++ index_lookup=index_lookup + ) + + def __init__( + self, + find_links: List[str], + index_urls: List[str], ++ index_lookup: Optional[Dict[str, List[str]]] = None, + ) -> None: + self.find_links = find_links + self.index_urls = index_urls ++ self.index_lookup = index_lookup if index_lookup else {} + + def get_formatted_locations(self) -> str: + lines = [] +@@ -123,4 +127,7 @@ class SearchScope: + loc = loc + '/' + return loc + +- return [mkurl_pypi_url(url) for url in self.index_urls] ++ index_urls = self.index_urls ++ if project_name in self.index_lookup: ++ index_urls = [self.index_lookup[project_name]] ++ return [mkurl_pypi_url(url) for url in index_urls]