From c9c359a17fc34e602d1d6cae912bd4073f7938fe Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sun, 13 Mar 2022 19:28:30 -0400 Subject: [PATCH 01/12] issue-4637 Do not consider extra_indexes when requirement.index is pinned. --- pipenv/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index c2041b3e..c2f40940 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1413,9 +1413,10 @@ def pip_install( ignore_hashes = False line = None # Try installing for each source in project.sources. - if not index and requirement.index: + if requirement.index: index = requirement.index - if index and not extra_indexes: + extra_indexes = [] + elif index and not extra_indexes: extra_indexes = list(project.sources) if requirement and requirement.vcs or requirement.editable: requirement.index = None From fdf3ed6ef63459b3f4618e62ae7f8641cf8135ae Mon Sep 17 00:00:00 2001 From: Matt Davis <=> Date: Sun, 13 Mar 2022 19:50:06 -0400 Subject: [PATCH 02/12] check pt changes --- pipenv/utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index d43158bb..0c7a17d7 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -496,6 +496,14 @@ class Resolver: # directories into the initial constraint pool to be resolved with the # rest of the dependencies, while adding the files/vcs deps/paths themselves # to the lockfile directly + if req.name in index_lookup: + use_sources = list(filter(lambda d: d['name'] == index_lookup[req.name], sources)) + else: + use_sources = sources + transient_resolver = cls( + [], req_dir, project, use_sources, index_lookup=index_lookup, + markers_lookup=markers_lookup, clear=clear, pre=pre + ) constraint_update, lockfile_update = cls.get_deps_from_req( req, resolver=transient_resolver, resolve_vcs=project.s.PIPENV_RESOLVE_VCS ) @@ -818,10 +826,13 @@ class Resolver: def parsed_constraints(self): from pipenv.vendor.pip_shims import shims + pip_options = self.pip_options + print("pip_options", pip_options) + pip_options.extra_index_urls = [] if self._parsed_constraints is None: self._parsed_constraints = shims.parse_requirements( self.constraint_file, finder=self.finder, session=self.session, - options=self.pip_options + options=pip_options ) return self._parsed_constraints From d677d628912f6e44e5538eb8a0903d1b6cb02d51 Mon Sep 17 00:00:00 2001 From: Matt Davis <=> Date: Sun, 13 Mar 2022 22:34:05 -0400 Subject: [PATCH 03/12] restrict pip install to the index that is specified. --- pipenv/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index c2f40940..4c154d37 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1416,8 +1416,9 @@ def pip_install( if requirement.index: index = requirement.index extra_indexes = [] - elif index and not extra_indexes: + if index and not extra_indexes: extra_indexes = list(project.sources) + extra_indexes = list(filter(lambda d: d['name'] == requirement.index, extra_indexes)) if requirement and requirement.vcs or requirement.editable: requirement.index = None # Install dependencies when a package is a non-editable VCS dependency. @@ -1437,6 +1438,7 @@ def pip_install( project, index, extra_indexes=extra_indexes, trusted_hosts=trusted_hosts, pypi_mirror=pypi_mirror ) + sources = list(filter(lambda d: d['name'] == requirement.index, sources)) if r: with open(r, "r") as fh: if "--hash" not in fh.read(): From bd5f43cbb825a14f083a8fa7a583a099fbc56e15 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 14 Mar 2022 03:34:25 -0400 Subject: [PATCH 04/12] This will respect the indexes but it required changing vendored code --- pipenv/core.py | 6 +++--- pipenv/patched/notpip/_internal/index/collector.py | 9 ++++++--- .../patched/notpip/_internal/models/search_scope.py | 11 +++++++++-- pipenv/utils.py | 12 ++++++++++-- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 4c154d37..83d99a0b 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1413,9 +1413,8 @@ def pip_install( ignore_hashes = False line = None # Try installing for each source in project.sources. - if requirement.index: + if not index and requirement.index: index = requirement.index - extra_indexes = [] if index and not extra_indexes: extra_indexes = list(project.sources) extra_indexes = list(filter(lambda d: d['name'] == requirement.index, extra_indexes)) @@ -1462,7 +1461,8 @@ def pip_install( pip_command.extend(["-r", vistir.path.normalize_path(r)]) elif line: pip_command.extend(line) - pip_command.extend(prepare_pip_source_args(sources)) + pip_source_args = prepare_pip_source_args(sources) + pip_command.extend(pip_source_args) if project.s.is_verbose(): click.echo(f"$ {cmd_list_to_shell(pip_command)}", err=True) cache_dir = Path(project.s.PIPENV_CACHE_DIR) diff --git a/pipenv/patched/notpip/_internal/index/collector.py b/pipenv/patched/notpip/_internal/index/collector.py index 98ac9d2e..7167ddc1 100644 --- a/pipenv/patched/notpip/_internal/index/collector.py +++ b/pipenv/patched/notpip/_internal/index/collector.py @@ -445,15 +445,18 @@ class LinkCollector: self, session: PipSession, search_scope: SearchScope, + index_lookup: dict = None, ) -> None: self.search_scope = search_scope self.session = session + self.index_lookup = index_lookup if index_lookup else {} @classmethod def create( cls, session: PipSession, options: Values, - suppress_no_index: bool = False + suppress_no_index: bool = False, + index_lookup: dict = None, ) -> "LinkCollector": """ :param session: The Session to use to make requests. @@ -472,10 +475,10 @@ class LinkCollector: find_links = options.find_links or [] search_scope = SearchScope.create( - find_links=find_links, index_urls=index_urls, + find_links=find_links, index_urls=index_urls, index_lookup=index_lookup ) link_collector = LinkCollector( - session=session, search_scope=search_scope, + session=session, search_scope=search_scope, index_lookup=index_lookup ) return link_collector diff --git a/pipenv/patched/notpip/_internal/models/search_scope.py b/pipenv/patched/notpip/_internal/models/search_scope.py index 4b700407..d21a5005 100644 --- a/pipenv/patched/notpip/_internal/models/search_scope.py +++ b/pipenv/patched/notpip/_internal/models/search_scope.py @@ -20,13 +20,14 @@ class SearchScope: Encapsulates the locations that pip is configured to search. """ - __slots__ = ["find_links", "index_urls"] + __slots__ = ["find_links", "index_urls", "index_lookup"] @classmethod def create( cls, find_links: List[str], index_urls: List[str], + index_lookup: dict = None, ) -> "SearchScope": """ Create a SearchScope object after normalizing the `find_links`. @@ -60,15 +61,18 @@ class SearchScope: return cls( find_links=built_find_links, index_urls=index_urls, + index_lookup=index_lookup ) def __init__( self, find_links: List[str], index_urls: List[str], + index_lookup: dict = None, ) -> None: self.find_links = find_links self.index_urls = index_urls + self.index_lookup = index_lookup if index_lookup else {} def get_formatted_locations(self) -> str: lines = [] @@ -123,4 +127,7 @@ class SearchScope: loc = loc + '/' return loc - return [mkurl_pypi_url(url) for url in self.index_urls] + index_urls = self.index_urls + if project_name in self.index_lookup: + index_urls = [self.index_lookup[project_name]] + return [mkurl_pypi_url(url) for url in index_urls] diff --git a/pipenv/utils.py b/pipenv/utils.py index 0c7a17d7..10e94bd8 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -804,6 +804,14 @@ class Resolver: options=self.pip_options, session=self.session ) + index_mapping = {} + for source in self.sources: + index_mapping[source['name']] = source['url'] + alt_index_lookup = {} + for req_name, index in self.index_lookup.items(): + alt_index_lookup[req_name] = index_mapping[index] + self._finder._link_collector.index_lookup = alt_index_lookup + self._finder._link_collector.search_scope.index_lookup = alt_index_lookup return self._finder @property @@ -827,7 +835,6 @@ class Resolver: from pipenv.vendor.pip_shims import shims pip_options = self.pip_options - print("pip_options", pip_options) pip_options.extra_index_urls = [] if self._parsed_constraints is None: self._parsed_constraints = shims.parse_requirements( @@ -887,9 +894,10 @@ class Resolver: from pipenv.vendor.pip_shims.shims import InstallationError from pipenv.exceptions import ResolutionFailure + constraints = self.constraints with temp_environ(), self.get_resolver() as resolver: try: - results = resolver.resolve(self.constraints, check_supported_wheels=False) + results = resolver.resolve(constraints, check_supported_wheels=False) except InstallationError as e: raise ResolutionFailure(message=str(e)) else: From e61403f802239e05c0c2a9d60b5c1375b4225809 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 14 Mar 2022 09:06:46 -0400 Subject: [PATCH 05/12] safety when doing these new index filters. --- pipenv/core.py | 2 +- pipenv/utils.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 83d99a0b..f6d8088b 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1437,7 +1437,7 @@ def pip_install( project, index, extra_indexes=extra_indexes, trusted_hosts=trusted_hosts, pypi_mirror=pypi_mirror ) - sources = list(filter(lambda d: d['name'] == requirement.index, sources)) + sources = list(filter(lambda d: d.get('name') == requirement.index, sources)) if r: with open(r, "r") as fh: if "--hash" not in fh.read(): diff --git a/pipenv/utils.py b/pipenv/utils.py index 10e94bd8..018b84f0 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -496,9 +496,10 @@ class Resolver: # directories into the initial constraint pool to be resolved with the # rest of the dependencies, while adding the files/vcs deps/paths themselves # to the lockfile directly + use_sources = None if req.name in index_lookup: - use_sources = list(filter(lambda d: d['name'] == index_lookup[req.name], sources)) - else: + use_sources = list(filter(lambda s: s.get('name') == index_lookup[req.name], sources)) + if not use_sources: use_sources = sources transient_resolver = cls( [], req_dir, project, use_sources, index_lookup=index_lookup, @@ -806,10 +807,12 @@ class Resolver: ) index_mapping = {} for source in self.sources: - index_mapping[source['name']] = source['url'] + if source.get('name'): + index_mapping[source['name']] = source['url'] alt_index_lookup = {} for req_name, index in self.index_lookup.items(): - alt_index_lookup[req_name] = index_mapping[index] + if index_mapping.get(index): + alt_index_lookup[req_name] = index_mapping[index] self._finder._link_collector.index_lookup = alt_index_lookup self._finder._link_collector.search_scope.index_lookup = alt_index_lookup return self._finder From e1d046623cc44a7f60037b4c2a339d2346a0074b Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 14 Mar 2022 10:00:52 -0400 Subject: [PATCH 06/12] resolve failing tests due to requirement.index being None --- pipenv/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index f6d8088b..9e5d3e48 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1417,7 +1417,8 @@ def pip_install( index = requirement.index if index and not extra_indexes: extra_indexes = list(project.sources) - extra_indexes = list(filter(lambda d: d['name'] == requirement.index, extra_indexes)) + if requirement.index: + extra_indexes = list(filter(lambda d: d['name'] == requirement.index, extra_indexes)) if requirement and requirement.vcs or requirement.editable: requirement.index = None # Install dependencies when a package is a non-editable VCS dependency. @@ -1437,7 +1438,8 @@ def pip_install( project, index, extra_indexes=extra_indexes, trusted_hosts=trusted_hosts, pypi_mirror=pypi_mirror ) - sources = list(filter(lambda d: d.get('name') == requirement.index, sources)) + if requirement.index in sources: + sources = list(filter(lambda d: d.get('name') == requirement.index, sources)) if r: with open(r, "r") as fh: if "--hash" not in fh.read(): From c50dc8017bedfd10b1372513b43474f5f23d62f7 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 14 Mar 2022 11:40:48 -0400 Subject: [PATCH 07/12] this part of the change was not required. --- pipenv/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 018b84f0..a96ab65d 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -897,10 +897,9 @@ class Resolver: from pipenv.vendor.pip_shims.shims import InstallationError from pipenv.exceptions import ResolutionFailure - constraints = self.constraints with temp_environ(), self.get_resolver() as resolver: try: - results = resolver.resolve(constraints, check_supported_wheels=False) + results = resolver.resolve(self.constraints, check_supported_wheels=False) except InstallationError as e: raise ResolutionFailure(message=str(e)) else: From fcdc402c821d6f5cfdbb75e9b24ff2fd598f4a37 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 14 Mar 2022 11:42:42 -0400 Subject: [PATCH 08/12] this part of the change was not required. --- pipenv/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 9e5d3e48..7aec3daa 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1463,8 +1463,7 @@ def pip_install( pip_command.extend(["-r", vistir.path.normalize_path(r)]) elif line: pip_command.extend(line) - pip_source_args = prepare_pip_source_args(sources) - pip_command.extend(pip_source_args) + pip_command.extend(prepare_pip_source_args(sources)) if project.s.is_verbose(): click.echo(f"$ {cmd_list_to_shell(pip_command)}", err=True) cache_dir = Path(project.s.PIPENV_CACHE_DIR) From bf00739895992b389b1060693da004d94d654fe2 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 14 Mar 2022 21:16:29 -0400 Subject: [PATCH 09/12] Seems to resolve the two tests that failed when run together as well as maintain the correct behavior of the issue-4637 example script --- pipenv/core.py | 6 ++++-- pipenv/utils.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 7aec3daa..c0f0840b 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1416,9 +1416,11 @@ def pip_install( if not index and requirement.index: index = requirement.index if index and not extra_indexes: - extra_indexes = list(project.sources) + extra_indexes = [] if requirement.index: - extra_indexes = list(filter(lambda d: d['name'] == requirement.index, extra_indexes)) + extra_indexes = list(filter(lambda d: d['name'] == requirement.index, project.sources)) + if not extra_indexes: + extra_indexes = list(project.sources) if requirement and requirement.vcs or requirement.editable: requirement.index = None # Install dependencies when a package is a non-editable VCS dependency. diff --git a/pipenv/utils.py b/pipenv/utils.py index a96ab65d..2abc0ccb 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -897,6 +897,7 @@ class Resolver: from pipenv.vendor.pip_shims.shims import InstallationError from pipenv.exceptions import ResolutionFailure + self.constraints # For some reason its important to evaluate constraints before resolver context with temp_environ(), self.get_resolver() as resolver: try: results = resolver.resolve(self.constraints, check_supported_wheels=False) From f6e5be201541a1a4be7c865a51fb688deb1a4fff Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sat, 19 Mar 2022 06:43:53 -0400 Subject: [PATCH 10/12] refactor type hints and add a patch for this change against the pip 21 version off main. --- .../notpip/_internal/index/collector.py | 5 +- .../notpip/_internal/models/search_scope.py | 6 +- .../patches/patched/pip_index_safety.patch | 102 ++++++++++++++++++ 3 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 tasks/vendoring/patches/patched/pip_index_safety.patch diff --git a/pipenv/patched/notpip/_internal/index/collector.py b/pipenv/patched/notpip/_internal/index/collector.py index 7167ddc1..fc532c40 100644 --- a/pipenv/patched/notpip/_internal/index/collector.py +++ b/pipenv/patched/notpip/_internal/index/collector.py @@ -17,6 +17,7 @@ from optparse import Values from typing import ( Callable, Iterable, + Dict, List, MutableMapping, NamedTuple, @@ -445,7 +446,7 @@ class LinkCollector: self, session: PipSession, search_scope: SearchScope, - index_lookup: dict = None, + index_lookup: Optional[Dict[str, List[str]]] = None, ) -> None: self.search_scope = search_scope self.session = session @@ -456,7 +457,7 @@ class LinkCollector: cls, session: PipSession, options: Values, suppress_no_index: bool = False, - index_lookup: dict = None, + index_lookup: Optional[Dict[str, List[str]]] = None, ) -> "LinkCollector": """ :param session: The Session to use to make requests. diff --git a/pipenv/patched/notpip/_internal/models/search_scope.py b/pipenv/patched/notpip/_internal/models/search_scope.py index d21a5005..58154c31 100644 --- a/pipenv/patched/notpip/_internal/models/search_scope.py +++ b/pipenv/patched/notpip/_internal/models/search_scope.py @@ -3,7 +3,7 @@ import logging import os import posixpath import urllib.parse -from typing import List +from typing import Dict, List, Optional from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name @@ -27,7 +27,7 @@ class SearchScope: cls, find_links: List[str], index_urls: List[str], - index_lookup: dict = None, + index_lookup: Optional[Dict[str, List[str]]] = None, ) -> "SearchScope": """ Create a SearchScope object after normalizing the `find_links`. @@ -68,7 +68,7 @@ class SearchScope: self, find_links: List[str], index_urls: List[str], - index_lookup: dict = None, + index_lookup: Optional[Dict[str, List[str]]] = None, ) -> None: self.find_links = find_links self.index_urls = index_urls diff --git a/tasks/vendoring/patches/patched/pip_index_safety.patch b/tasks/vendoring/patches/patched/pip_index_safety.patch new file mode 100644 index 00000000..3eea4dc1 --- /dev/null +++ b/tasks/vendoring/patches/patched/pip_index_safety.patch @@ -0,0 +1,102 @@ +diff --git a/pipenv/patched/pip/_internal/index/collector.py b/pipenv/patched/pip/_internal/index/collector.py +index 98ac9d2e..fc532c40 100644 +--- a/pipenv/patched/pip/_internal/index/collector.py ++++ b/pipenv/patched/pip/_internal/index/collector.py +@@ -17,6 +17,7 @@ from optparse import Values + from typing import ( + Callable, + Iterable, ++ Dict, + List, + MutableMapping, + NamedTuple, +@@ -445,15 +446,18 @@ class LinkCollector: + self, + session: PipSession, + search_scope: SearchScope, ++ index_lookup: Optional[Dict[str, List[str]]] = None, + ) -> None: + self.search_scope = search_scope + self.session = session ++ self.index_lookup = index_lookup if index_lookup else {} + + @classmethod + def create( + cls, session: PipSession, + options: Values, +- suppress_no_index: bool = False ++ suppress_no_index: bool = False, ++ index_lookup: Optional[Dict[str, List[str]]] = None, + ) -> "LinkCollector": + """ + :param session: The Session to use to make requests. +@@ -472,10 +476,10 @@ class LinkCollector: + find_links = options.find_links or [] + + search_scope = SearchScope.create( +- find_links=find_links, index_urls=index_urls, ++ find_links=find_links, index_urls=index_urls, index_lookup=index_lookup + ) + link_collector = LinkCollector( +- session=session, search_scope=search_scope, ++ session=session, search_scope=search_scope, index_lookup=index_lookup + ) + return link_collector + +diff --git a/pipenv/patched/pip/_internal/models/search_scope.py b/pipenv/patched/pip/_internal/models/search_scope.py +index 4b700407..58154c31 100644 +--- a/pipenv/patched/pip/_internal/models/search_scope.py ++++ b/pipenv/patched/pip/_internal/models/search_scope.py +@@ -3,7 +3,7 @@ import logging + import os + import posixpath + import urllib.parse +-from typing import List ++from typing import Dict, List, Optional + + from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name + +@@ -20,13 +20,14 @@ class SearchScope: + Encapsulates the locations that pip is configured to search. + """ + +- __slots__ = ["find_links", "index_urls"] ++ __slots__ = ["find_links", "index_urls", "index_lookup"] + + @classmethod + def create( + cls, + find_links: List[str], + index_urls: List[str], ++ index_lookup: Optional[Dict[str, List[str]]] = None, + ) -> "SearchScope": + """ + Create a SearchScope object after normalizing the `find_links`. +@@ -60,15 +61,18 @@ class SearchScope: + return cls( + find_links=built_find_links, + index_urls=index_urls, ++ index_lookup=index_lookup + ) + + def __init__( + self, + find_links: List[str], + index_urls: List[str], ++ index_lookup: Optional[Dict[str, List[str]]] = None, + ) -> None: + self.find_links = find_links + self.index_urls = index_urls ++ self.index_lookup = index_lookup if index_lookup else {} + + def get_formatted_locations(self) -> str: + lines = [] +@@ -123,4 +127,7 @@ class SearchScope: + loc = loc + '/' + return loc + +- return [mkurl_pypi_url(url) for url in self.index_urls] ++ index_urls = self.index_urls ++ if project_name in self.index_lookup: ++ index_urls = [self.index_lookup[project_name]] ++ return [mkurl_pypi_url(url) for url in index_urls] From ceb66b702534813d5a3fb2298c20c54bf926f94d Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sat, 19 Mar 2022 06:54:46 -0400 Subject: [PATCH 11/12] Add news fragment. --- news/4637.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/4637.bugfix.rst diff --git a/news/4637.bugfix.rst b/news/4637.bugfix.rst new file mode 100644 index 00000000..a51906bd --- /dev/null +++ b/news/4637.bugfix.rst @@ -0,0 +1 @@ +Patched our vendored Pip to fix: Pipenv Lock (Or Install) Does Not Respect Index Specified For A Package. From 0044bfaee7a8be42b807d6ac2b0a6e7c0649f415 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sat, 19 Mar 2022 07:31:25 -0400 Subject: [PATCH 12/12] Fix patch vendoring build. --- tasks/vendoring/patches/patched/pip_index_safety.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/vendoring/patches/patched/pip_index_safety.patch b/tasks/vendoring/patches/patched/pip_index_safety.patch index 3eea4dc1..91b2235b 100644 --- a/tasks/vendoring/patches/patched/pip_index_safety.patch +++ b/tasks/vendoring/patches/patched/pip_index_safety.patch @@ -54,7 +54,7 @@ index 4b700407..58154c31 100644 -from typing import List +from typing import Dict, List, Optional - from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name + from pip._vendor.packaging.utils import canonicalize_name @@ -20,13 +20,14 @@ class SearchScope: Encapsulates the locations that pip is configured to search.