From 00dd84537244262e98db1fa45319d33254f383e7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 23 Jul 2018 12:38:50 -0400 Subject: [PATCH] Enable parsing of dependency links in setup.py - Fixed errors with url parsing during hashing Signed-off-by: Dan Ryan --- news/2434.bugfix | 1 + pipenv/patched/piptools/repositories/pypi.py | 14 ++++--- .../vendoring/patches/patched/piptools.patch | 38 +++++++++++-------- 3 files changed, 33 insertions(+), 20 deletions(-) create mode 100644 news/2434.bugfix diff --git a/news/2434.bugfix b/news/2434.bugfix new file mode 100644 index 00000000..0a9603e1 --- /dev/null +++ b/news/2434.bugfix @@ -0,0 +1 @@ +Fixed the ability of pipenv to parse ``dependency_links`` from ``setup.py`` when ``PIP_PROCESS_DEPENDENCY_LINKS`` is enabled. diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index f09ff372..2f746094 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,7 +1,7 @@ # coding: utf-8 from __future__ import (absolute_import, division, print_function, unicode_literals) - +import copy import hashlib import os import sys @@ -64,15 +64,19 @@ class HashCache(SafeFileCache): def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - can_hash = location.hash + vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') + new_location = copy.deepcopy(location) + if any(new_location.url.startswith(vcs) for vcs in vcs_uris): + new_location.url = new_location.url.split("+", 1)[-1] + can_hash = new_location.hash if can_hash: # hash url WITH fragment - hash_value = self.get(location.url) + hash_value = self.get(new_location.url) if not hash_value: - hash_value = self._get_file_hash(location) + hash_value = self._get_file_hash(new_location) hash_value = hash_value.encode('utf8') if can_hash: - self.set(location.url, hash_value) + self.set(new_location.url, hash_value) return hash_value.decode('utf8') def _get_file_hash(self, location): diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index f5fb822f..56696e08 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,11 +19,15 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..c922be1 100644 +index 1c4b943..245e9ce 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, - +@@ -1,9 +1,10 @@ + # coding: utf-8 + from __future__ import (absolute_import, division, print_function, + unicode_literals) +- ++import copy import hashlib import os +import sys @@ -58,7 +62,7 @@ index 1c4b943..c922be1 100644 from .base import BaseRepository -@@ -37,6 +49,40 @@ except ImportError: +@@ -37,6 +49,44 @@ except ImportError: from pip.wheel import WheelCache @@ -77,15 +81,19 @@ index 1c4b943..c922be1 100644 + def get_hash(self, location): + # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it + hash_value = None -+ can_hash = location.hash ++ vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') ++ new_location = copy.deepcopy(location) ++ if any(new_location.url.startswith(vcs) for vcs in vcs_uris): ++ new_location.url = new_location.url.split("+", 1)[-1] ++ can_hash = new_location.hash + if can_hash: + # hash url WITH fragment -+ hash_value = self.get(location.url) ++ hash_value = self.get(new_location.url) + if not hash_value: -+ hash_value = self._get_file_hash(location) ++ hash_value = self._get_file_hash(new_location) + hash_value = hash_value.encode('utf8') + if can_hash: -+ self.set(location.url, hash_value) ++ self.set(new_location.url, hash_value) + return hash_value.decode('utf8') + + def _get_file_hash(self, location): @@ -99,7 +107,7 @@ index 1c4b943..c922be1 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,10 +92,11 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +96,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -113,7 +121,7 @@ index 1c4b943..c922be1 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -74,11 +121,15 @@ class PyPIRepository(BaseRepository): +@@ -74,11 +125,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -131,7 +139,7 @@ index 1c4b943..c922be1 100644 def freshen_build_caches(self): """ -@@ -114,10 +165,14 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +169,14 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -148,7 +156,7 @@ index 1c4b943..c922be1 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +181,71 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +185,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -223,7 +231,7 @@ index 1c4b943..c922be1 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,20 +270,40 @@ class PyPIRepository(BaseRepository): +@@ -155,20 +274,40 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -268,7 +276,7 @@ index 1c4b943..c922be1 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +323,97 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +327,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -369,7 +377,7 @@ index 1c4b943..c922be1 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -217,24 +432,22 @@ class PyPIRepository(BaseRepository): +@@ -217,24 +436,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint.