From 92c7328d14e09a888fb366811a0698def2cf0192 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 16 May 2018 23:28:29 -0400 Subject: [PATCH 1/7] Drop cacert.pem copy task from vendor script Signed-off-by: Dan Ryan --- tasks/vendoring/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 5272a049..88675cff 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -309,11 +309,6 @@ def vendor(ctx, vendor_dir, rewrite=True): log('Renaming specified libs...') for item in vendor_dir.iterdir(): if item.is_dir(): - if item.name == 'requests' and not (item / 'cacert.pem').exists(): - if 'certifi' in vendored_libs: - cert = vendor_dir / 'certifi' / 'cacert.pem' - copy_to = item / 'cacert.pem' - copy_to.write_bytes(cert.read_bytes()) if rewrite: log('Rewriting imports for %s...' % item) rewrite_imports(item, vendored_libs, vendor_dir) From 45dc5d10ec38dd0f7c4eed92f63f8e00f3fdc053 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 16 May 2018 23:35:11 -0400 Subject: [PATCH 2/7] Update patched version of pip-tools for vendoring Signed-off-by: Dan Ryan --- pipenv/patched/patched.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index ef7f862f..4fc8f274 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -3,6 +3,6 @@ git+https://github.com/jumpscale7/python-consistent-toml.git#egg=contoml crayons==0.1.2 git+https://github.com/berdario/pew.git@1.1.5#egg=pew pipfile==0.0.2 -pip-tools==2.0.1 +git+https://github.com/jazzband/pip-tools.git@9cb41d828fcb0967a32cc140c1dcaca94e5f4daa#egg=piptools prettytoml==0.3 pip==9.0.3 From 0dbc5c687143395644c981b59b379c8d36232cde Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 17 May 2018 10:50:54 -0400 Subject: [PATCH 3/7] Update piptools patch Signed-off-by: Dan Ryan --- .../vendoring/patches/patched/piptools.patch | 265 ++++++++---------- 1 file changed, 122 insertions(+), 143 deletions(-) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 951ce87d..132687d9 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -1,14 +1,14 @@ diff --git a/pipenv/patched/piptools/locations.py b/pipenv/patched/piptools/locations.py -index aa0610b..5791f0f 100644 +index 4e6174c..75f9b49 100644 --- a/pipenv/patched/piptools/locations.py +++ b/pipenv/patched/piptools/locations.py @@ -2,10 +2,13 @@ import os from shutil import rmtree from .click import secho --from pip.utils.appdirs import user_cache_dir +-from ._compat import user_cache_dir +# Patch by vphilippon 2017-11-22: Use pipenv cache path. -+# from pip9.utils.appdirs import user_cache_dir ++# from ._compat import user_cache_dir +from pipenv.environments import PIPENV_CACHE_DIR # The user_cache_dir helper comes straight from pip itself @@ -19,51 +19,45 @@ index aa0610b..5791f0f 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index d3b7fe7..e1f63d2 100644 +index 1c4b943..39a285b 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -7,20 +7,59 @@ import os - from contextlib import contextmanager - from shutil import rmtree +@@ -15,10 +15,16 @@ from .._compat import ( + Wheel, + FAVORITE_HASH, + TemporaryDirectory, +- PyPI ++ PyPI, ++ InstallRequirement, ++ SafeFileCache, + ) --from pip.download import is_file_url, url_to_path --from pip.index import PackageFinder --from pip.req.req_set import RequirementSet --from pip.wheel import Wheel --from pip.utils.hashes import FAVORITE_HASH -+from notpip.download import is_file_url, url_to_path -+from notpip.index import PackageFinder -+from notpip.req.req_set import RequirementSet -+from notpip.wheel import Wheel -+from notpip.req.req_install import InstallRequirement +from pip9._vendor.packaging.requirements import InvalidRequirement +from pip9._vendor.pyparsing import ParseException -+from notpip.download import SafeFileCache -+from notpip.utils.hashes import FAVORITE_HASH - - from .._compat import TemporaryDirectory ++ from ..cache import CACHE_DIR +from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound from ..utils import (fs_str, is_pinned_requirement, lookup_table, make_install_requirement) - from .base import BaseRepository +@@ -37,6 +43,40 @@ except ImportError: + from pip.wheel import WheelCache +class HashCache(SafeFileCache): + """Caches hashes of PyPI artifacts so we do not need to re-download them + -+ Hashes are only cached when the URL appears to contain a hash in it (and the cache key includes -+ the hash value returned from the server). This ought to avoid issues where the location on the ++ Hashes are only cached when the URL appears to contain a hash in it and the cache key includes ++ the hash value returned from the server). This ought to avoid ssues where the location on the + server changes.""" + def __init__(self, *args, **kwargs): + session = kwargs.pop('session') + self.session = session -+ kwargs.setdefault('directory', os.path.join(PIPENV_CACHE_DIR, 'hash-cache')) ++ kwargs.setdefault('directory', os.path.join(PIPENV_CACHE_DIR, hash-cache')) + super(HashCache, self).__init__(*args, **kwargs) + + def get_hash(self, location): -+ # if there is no location hash (i.e., md5 / sha256 / etc) we don't want to store it ++ # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it + hash_value = None + can_hash = location.hash + if can_hash: @@ -85,10 +79,9 @@ index d3b7fe7..e1f63d2 100644 + + class PyPIRepository(BaseRepository): -- DEFAULT_INDEX_URL = 'https://pypi.python.org/simple' -+ DEFAULT_INDEX_URL = 'https://pypi.org/simple' + DEFAULT_INDEX_URL = PyPI.simple_url -@@ -30,8 +69,9 @@ class PyPIRepository(BaseRepository): +@@ -46,8 +86,9 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -96,10 +89,10 @@ index d3b7fe7..e1f63d2 100644 + def __init__(self, pip_options, session, use_json=False): self.session = session + self.use_json = use_json + self.pip_options = pip_options + self.wheel_cache = WheelCache(CACHE_DIR, pip_options.format_control) - index_urls = [pip_options.index_url] + pip_options.extra_index_urls - if pip_options.no_index: -@@ -56,6 +96,10 @@ class PyPIRepository(BaseRepository): +@@ -74,6 +115,10 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -110,7 +103,7 @@ index d3b7fe7..e1f63d2 100644 # Setup file paths self.freshen_build_caches() -@@ -108,11 +152,60 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +171,60 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -128,7 +121,7 @@ index d3b7fe7..e1f63d2 100644 + def get_json_dependencies(self, ireq): + + if not (is_pinned_requirement(ireq)): -+ raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq)) ++ raise TypeError('Expected pinned InstallRequirement, got {}.format(ireq)) + + def gen(ireq): + if self.DEFAULT_INDEX_URL in self.finder.index_urls: @@ -139,8 +132,9 @@ index d3b7fe7..e1f63d2 100644 + # TODO: Latest isn't always latest. + latest = list(r.json()['releases'].keys())[-1] + if str(ireq.req.specifier) == '=={0}'.format(latest): -+ -+ for requires in r.json().get('info', {}).get('requires_dist', {}): ++ latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest) ++ latest_requires = self.session.get(latest_url) ++ for requires in latest_requires.json().get('info', {}).get('requires_dist', {}): + i = InstallRequirement.from_line(requires) + + if 'extra' not in repr(i.markers): @@ -153,7 +147,6 @@ index d3b7fe7..e1f63d2 100644 + return set(self._json_dep_cache[ireq]) + except Exception: + return set() -+ def get_dependencies(self, ireq): + json_results = set() @@ -174,7 +167,7 @@ index d3b7fe7..e1f63d2 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -121,6 +214,19 @@ class PyPIRepository(BaseRepository): +@@ -139,6 +233,18 @@ class PyPIRepository(BaseRepository): if not (ireq.editable or is_pinned_requirement(ireq)): raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq)) @@ -189,24 +182,36 @@ index d3b7fe7..e1f63d2 100644 + ) + except TypeError: + pass -+ + if ireq not in self._dependencies_cache: if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # No download_dir for locally available editable requirements. -@@ -142,8 +248,43 @@ class PyPIRepository(BaseRepository): - self.source_dir, - download_dir=download_dir, - wheel_download_dir=self._wheel_download_dir, -- session=self.session) -- self._dependencies_cache[ireq] = reqset._prepare_file(self.finder, ireq) -+ session=self.session, -+ ignore_installed=True, -+ ignore_compatibility=False -+ ) -+ -+ result = reqset._prepare_file(self.finder, ireq, ignore_requires_python=True) -+ +@@ -164,11 +270,14 @@ class PyPIRepository(BaseRepository): + download_dir=download_dir, + wheel_download_dir=self._wheel_download_dir, + session=self.session, ++ ignore_installed=True, ++ ignore_compatibility=False, + wheel_cache=self.wheel_cache, + ) +- self._dependencies_cache[ireq] = reqset._prepare_file( ++ result = reqset._prepare_file( + self.finder, +- ireq ++ ireq, ++ ignore_requires_python=True + ) + except TypeError: + # Pip >= 10 (new resolver!) +@@ -195,9 +304,39 @@ class PyPIRepository(BaseRepository): + isolated=False, + wheel_cache=self.wheel_cache, + use_user_site=False, ++ ignore_compatibility=False + ) + self.resolver.resolve(reqset) +- self._dependencies_cache[ireq] = reqset.requirements.values() ++ result = reqset.requirements.values() + # Convert setup_requires dict into a somewhat usable form. + if setup_requires: + for section in setup_requires: @@ -231,16 +236,15 @@ index d3b7fe7..e1f63d2 100644 + pass + + if reqset.requires_python: -+ + marker = 'python_version=="{0}"'.format(reqset.requires_python.replace(' ', '')) + new_req = InstallRequirement.from_line('{0}; {1}'.format(str(ireq.req), marker)) + result = [new_req] + + self._dependencies_cache[ireq] = result + reqset.cleanup_files() return set(self._dependencies_cache[ireq]) - def get_hashes(self, ireq): -@@ -169,17 +310,10 @@ class PyPIRepository(BaseRepository): +@@ -224,17 +363,10 @@ class PyPIRepository(BaseRepository): matching_candidates = candidates_by_version[matching_versions[0]] return { @@ -259,15 +263,6 @@ index d3b7fe7..e1f63d2 100644 @contextmanager def allow_all_wheels(self): """ -@@ -217,7 +351,7 @@ def open_local_or_remote_file(link, session): - """ - Open local or remote file for reading. - -- :type link: pip.index.Link -+ :type link: pip9.index.Link - :type session: requests.Session - :raises ValueError: If link points to a local directory. - :return: a context manager to the opened file-like object diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 8c4e981..862be14 100644 --- a/pipenv/patched/piptools/resolver.py @@ -361,86 +356,11 @@ index 8c4e981..862be14 100644 def reverse_dependencies(self, ireqs): non_editable = [ireq for ireq in ireqs if not ireq.editable] -diff --git a/pipenv/patched/piptools/scripts/compile.py b/pipenv/patched/piptools/scripts/compile.py -index b41f8b2..0e83bfc 100644 ---- a/pipenv/patched/piptools/scripts/compile.py -+++ b/pipenv/patched/piptools/scripts/compile.py -@@ -7,8 +7,8 @@ import os - import sys - import tempfile - --import pip --from pip.req import InstallRequirement, parse_requirements -+import pip9 -+from pip9.req import InstallRequirement, parse_requirements - - from .. import click - from ..exceptions import PipToolsError -@@ -21,7 +21,7 @@ from ..writer import OutputWriter - DEFAULT_REQUIREMENTS_FILE = 'requirements.in' - - --class PipCommand(pip.basecommand.Command): -+class PipCommand(pip9.basecommand.Command): - name = 'PipCommand' - - -@@ -251,8 +251,8 @@ def get_pip_command(): - # General options (find_links, index_url, extra_index_url, trusted_host, - # and pre) are defered to pip. - pip_command = PipCommand() -- index_opts = pip.cmdoptions.make_option_group( -- pip.cmdoptions.index_group, -+ index_opts = pip9.cmdoptions.make_option_group( -+ pip9.cmdoptions.index_group, - pip_command.parser, - ) - pip_command.parser.insert_option_group(0, index_opts) -diff --git a/pipenv/patched/piptools/scripts/sync.py b/pipenv/patched/piptools/scripts/sync.py -index 0f74370..c43c5af 100644 ---- a/pipenv/patched/piptools/scripts/sync.py -+++ b/pipenv/patched/piptools/scripts/sync.py -@@ -5,7 +5,7 @@ from __future__ import (absolute_import, division, print_function, - import os - import sys - --import pip -+import pip9 - - from .. import click, sync - from ..exceptions import PipToolsError -@@ -45,7 +45,7 @@ def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet, - log.error('ERROR: ' + msg) - sys.exit(2) - -- requirements = flat_map(lambda src: pip.req.parse_requirements(src, session=True), -+ requirements = flat_map(lambda src: pip9.req.parse_requirements(src, session=True), - src_files) - - try: -@@ -54,7 +54,7 @@ def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet, - log.error(str(e)) - sys.exit(2) - -- installed_dists = pip.get_installed_distributions(skip=[], user_only=user_only) -+ installed_dists = pip9.get_installed_distributions(skip=[], user_only=user_only) - to_install, to_uninstall = sync.diff(requirements, installed_dists) - - install_flags = [] diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py -index c67d1b5..db8bb9b 100644 +index fde5816..d76695d 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py -@@ -6,7 +6,7 @@ import sys - from itertools import chain, groupby - from collections import OrderedDict - --from pip.req import InstallRequirement -+from pip9.req import InstallRequirement - - from first import first - -@@ -41,16 +41,21 @@ def comment(text): +@@ -43,16 +43,21 @@ def comment(text): return style(text, fg='green') @@ -466,7 +386,7 @@ index c67d1b5..db8bb9b 100644 def format_requirement(ireq, marker=None): -@@ -64,7 +69,7 @@ def format_requirement(ireq, marker=None): +@@ -66,7 +71,7 @@ def format_requirement(ireq, marker=None): line = str(ireq.req).lower() if marker: @@ -475,4 +395,63 @@ index c67d1b5..db8bb9b 100644 return line - +diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py +index 7e8cdf3..2b5c12a 100644 +--- a/pipenv/patched/piptools/_compat/pip_compat.py ++++ b/pipenv/patched/piptools/_compat/pip_compat.py +@@ -1,30 +1,40 @@ + # -*- coding=utf-8 -*- + import importlib + +-def do_import(module_path, subimport=None, old_path=None): ++ ++def do_import(module_path, subimport=None, old_path=None, vendored_name=None): + internal = 'pip._internal.{0}'.format(module_path) + old_path = old_path or module_path + pip9 = 'pip.{0}'.format(old_path) +- try: +- _tmp = importlib.import_module(internal) +- except ImportError: +- _tmp = importlib.import_module(pip9) ++ _tmp = None ++ if vendored_name: ++ vendor = '{0}.{1}'.format(vendored_name, old_path) ++ try: ++ _tmp = importlib.import_module(vendor) ++ except ImportError: ++ pass ++ if not _tmp: ++ try: ++ _tmp = importlib.import_module(internal) ++ except ImportError: ++ _tmp = importlib.import_module(pip9) + if subimport: + return getattr(_tmp, subimport, _tmp) + return _tmp + + +-InstallRequirement = do_import('req.req_install', 'InstallRequirement') +-parse_requirements = do_import('req.req_file', 'parse_requirements') +-RequirementSet = do_import('req.req_set', 'RequirementSet') +-user_cache_dir = do_import('utils.appdirs', 'user_cache_dir') +-FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH') +-is_file_url = do_import('download', 'is_file_url') +-url_to_path = do_import('download', 'url_to_path') +-PackageFinder = do_import('index', 'PackageFinder') ++InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name='notpip') ++parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name='notpip') ++RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name='notpip') ++user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name='notpip') ++FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name='notpip') ++is_file_url = do_import('download', 'is_file_url', vendored_name='notpip') ++url_to_path = do_import('download', 'url_to_path', vendored_name='notpip') ++PackageFinder = do_import('index', 'PackageFinder', vendored_name='notpip') + FormatControl = do_import('index', 'FormatControl') +-Wheel = do_import('wheel', 'Wheel') ++Wheel = do_import('wheel', 'Wheel', vendored_name='notpip') + Command = do_import('basecommand', 'Command') + cmdoptions = do_import('cmdoptions') + get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') +-PyPI = do_import('models.index', 'PyPI') ++PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') ++SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') From 9986c75d68978e85d4ea9f8099b6c947e8d4bf7e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 17 May 2018 14:27:07 -0400 Subject: [PATCH 4/7] Update patches Signed-off-by: Dan Ryan --- .../patched/_post-pip-update-imports.patch | 92 ------------------- .../vendoring/patches/patched/piptools.patch | 21 +++-- tasks/vendoring/patches/vendor/pip.patch | 11 +++ 3 files changed, 26 insertions(+), 98 deletions(-) diff --git a/tasks/vendoring/patches/patched/_post-pip-update-imports.patch b/tasks/vendoring/patches/patched/_post-pip-update-imports.patch index fa32eb58..f1e74b81 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-imports.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-imports.patch @@ -174,95 +174,3 @@ index 4254fbd..a7efaa0 100644 DecodeError, ReadTimeoutError, ProtocolError, LocationParseError) from io import UnsupportedOperation -diff --git a/pipenv/patched/piptools/cache.py b/pipenv/patched/piptools/cache.py -index 7595b96..5a44953 100644 ---- a/pipenv/patched/piptools/cache.py -+++ b/pipenv/patched/piptools/cache.py -@@ -6,7 +6,7 @@ import json - import os - import sys - --from pip._vendor.packaging.requirements import Requirement -+from pip9._vendor.packaging.requirements import Requirement - - from .exceptions import PipToolsError - from .locations import CACHE_DIR -diff --git a/pipenv/patched/piptools/repositories/base.py b/pipenv/patched/piptools/repositories/base.py -index 57e85fd..69835c0 100644 ---- a/pipenv/patched/piptools/repositories/base.py -+++ b/pipenv/patched/piptools/repositories/base.py -@@ -44,5 +44,5 @@ class BaseRepository(object): - @contextmanager - def allow_all_wheels(self): - """ -- Monkey patches pip.Wheel to allow wheels from all platforms and Python versions. -+ Monkey patches pip9.Wheel to allow wheels from all platforms and Python versions. - """ -diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py -index 8f6f028..3c6182b 100644 ---- a/pipenv/patched/piptools/repositories/local.py -+++ b/pipenv/patched/piptools/repositories/local.py -@@ -6,7 +6,7 @@ from contextlib import contextmanager - - from piptools.utils import as_tuple, key_from_req, make_install_requirement - from .base import BaseRepository --from pip.utils.hashes import FAVORITE_HASH -+from pip9.utils.hashes import FAVORITE_HASH - - - def ireq_satisfied_by_existing_pin(ireq, existing_pin): -diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index e1f63d2..03833a4 100644 ---- a/pipenv/patched/piptools/repositories/pypi.py -+++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -317,7 +317,7 @@ class PyPIRepository(BaseRepository): - @contextmanager - def allow_all_wheels(self): - """ -- Monkey patches pip.Wheel to allow wheels from all platforms and Python versions. -+ Monkey patches pip9.Wheel to allow wheels from all platforms and Python versions. - - This also saves the candidate cache and set a new one, or else the results from the - previous non-patched calls will interfere. -diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index 862be14..64b1199 100644 ---- a/pipenv/patched/piptools/resolver.py -+++ b/pipenv/patched/piptools/resolver.py -@@ -8,7 +8,7 @@ from itertools import chain, count - import os - - from first import first --from pip.req import InstallRequirement -+from pip9.req import InstallRequirement - - from . import click - from .cache import DependencyCache -diff --git a/pipenv/patched/piptools/scripts/compile.py b/pipenv/patched/piptools/scripts/compile.py -index 0e83bfc..8194fcf 100644 ---- a/pipenv/patched/piptools/scripts/compile.py -+++ b/pipenv/patched/piptools/scripts/compile.py -@@ -247,9 +247,9 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, - - - def get_pip_command(): -- # Use pip's parser for pip.conf management and defaults. -+ # Use pip's parser for pip9.conf management and defaults. - # General options (find_links, index_url, extra_index_url, trusted_host, -- # and pre) are defered to pip. -+ # and pre) are defered to pip9. - pip_command = PipCommand() - index_opts = pip9.cmdoptions.make_option_group( - pip9.cmdoptions.index_group, -diff --git a/pipenv/patched/safety/cli.py b/pipenv/patched/safety/cli.py -index 37ae3ac..fc543ef 100644 ---- a/pipenv/patched/safety/cli.py -+++ b/pipenv/patched/safety/cli.py -@@ -15,7 +15,7 @@ try: - from pip import get_installed_distributions - except ImportError: - # pip 10 -- from pip._internal.utils.misc import get_installed_distributions -+ from pip9._internal.utils.misc import get_installed_distributions - - - @click.group() diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 132687d9..6f14002e 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -53,7 +53,7 @@ index 1c4b943..39a285b 100644 + def __init__(self, *args, **kwargs): + session = kwargs.pop('session') + self.session = session -+ kwargs.setdefault('directory', os.path.join(PIPENV_CACHE_DIR, hash-cache')) ++ kwargs.setdefault('directory', os.path.join(PIPENV_CACHE_DIR, 'hash-cache')) + super(HashCache, self).__init__(*args, **kwargs) + + def get_hash(self, location): @@ -121,7 +121,7 @@ index 1c4b943..39a285b 100644 + def get_json_dependencies(self, ireq): + + if not (is_pinned_requirement(ireq)): -+ raise TypeError('Expected pinned InstallRequirement, got {}.format(ireq)) ++ raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq)) + + def gen(ireq): + if self.DEFAULT_INDEX_URL in self.finder.index_urls: @@ -264,7 +264,7 @@ index 1c4b943..39a285b 100644 def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index 8c4e981..862be14 100644 +index 05ec8fd..6380e8f 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -14,7 +14,7 @@ from . import click @@ -327,7 +327,7 @@ index 8c4e981..862be14 100644 elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) -@@ -283,14 +295,26 @@ class Resolver(object): +@@ -283,14 +295,25 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) @@ -339,7 +339,7 @@ index 8c4e981..862be14 100644 dependency_strings = self.dependency_cache[ireq] log.debug(' {:25} requires {}'.format(format_requirement(ireq), ', '.join(sorted(dependency_strings, key=lambda s: s.lower())) or '-')) -+ from notpip._vendor.packaging.markers import InvalidMarker ++ from pip9._vendor.packaging.markers import InvalidMarker for dependency_string in dependency_strings: - yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) + try: @@ -352,7 +352,6 @@ index 8c4e981..862be14 100644 + yield InstallRequirement.from_line(_dependency_string, constraint=ireq.constraint) + except InvalidMarker: + yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) -+ def reverse_dependencies(self, ireqs): non_editable = [ireq for ireq in ireqs if not ireq.editable] @@ -455,3 +454,13 @@ index 7e8cdf3..2b5c12a 100644 -PyPI = do_import('models.index', 'PyPI') +PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') +SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') +diff --git a/pipenv/patched/piptools/_compat/__init__.py b/piptools/_compat/__init__.py +index 674674a..4259028 100644 +--- a/pipenv/patched/piptools/_compat/__init__.py ++++ b/pipenv/patched/piptools/_compat/__init__.py +@@ -27,4 +27,5 @@ from .pip_compat import ( + cmdoptions, + get_installed_distributions, + PyPI, ++ SafeFileCache, + ) diff --git a/tasks/vendoring/patches/vendor/pip.patch b/tasks/vendoring/patches/vendor/pip.patch index b9a4de4c..fc1e202e 100644 --- a/tasks/vendoring/patches/vendor/pip.patch +++ b/tasks/vendoring/patches/vendor/pip.patch @@ -351,3 +351,14 @@ index c9ebbf1..728f5cc 100644 path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir + +diff --git a/pipenv/vendor/pip9/__main__.py b/pipenv/vendor/pip9/__main__.py +index 9849a65c..da041f92 100644 +--- a/pipenv/vendor/pip9/__main__.py ++++ b/pipenv/vendor/pip9/__main__.py +@@ -16,4 +16,4 @@ if __package__ == '': + import pip9 # noqa + + if __name__ == '__main__': +- sys.exit(pip.main()) ++ sys.exit(pip9.main()) From 7151583f1ca69e55b82c48ff6ca16e3025fa2b0c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 17 May 2018 14:29:07 -0400 Subject: [PATCH 5/7] Revendor piptools for update Signed-off-by: Dan Ryan --- pipenv/patched/piptools/LICENSE.txt | 17 +++ pipenv/patched/piptools/__init__.py | 6 - pipenv/patched/piptools/_compat/__init__.py | 18 +++ pipenv/patched/piptools/_compat/pip_compat.py | 40 ++++++ pipenv/patched/piptools/cache.py | 2 +- pipenv/patched/piptools/exceptions.py | 34 +++++- pipenv/patched/piptools/locations.py | 2 +- pipenv/patched/piptools/repositories/base.py | 2 +- pipenv/patched/piptools/repositories/local.py | 2 +- pipenv/patched/piptools/repositories/pypi.py | 115 +++++++++++++----- pipenv/patched/piptools/resolver.py | 5 +- pipenv/patched/piptools/scripts/compile.py | 20 +-- pipenv/patched/piptools/scripts/sync.py | 6 +- pipenv/patched/piptools/utils.py | 18 ++- pipenv/patched/safety/cli.py | 2 +- 15 files changed, 227 insertions(+), 62 deletions(-) create mode 100644 pipenv/patched/piptools/LICENSE.txt create mode 100644 pipenv/patched/piptools/_compat/pip_compat.py diff --git a/pipenv/patched/piptools/LICENSE.txt b/pipenv/patched/piptools/LICENSE.txt new file mode 100644 index 00000000..89de3547 --- /dev/null +++ b/pipenv/patched/piptools/LICENSE.txt @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/patched/piptools/__init__.py b/pipenv/patched/piptools/__init__.py index f332ac15..e69de29b 100644 --- a/pipenv/patched/piptools/__init__.py +++ b/pipenv/patched/piptools/__init__.py @@ -1,6 +0,0 @@ -import os -import sys - -# Inject vendored directory into system path. -v_path = os.path.abspath(os.path.sep.join([os.path.dirname(os.path.realpath(__file__)), '_vendored'])) -sys.path.insert(0, v_path) diff --git a/pipenv/patched/piptools/_compat/__init__.py b/pipenv/patched/piptools/_compat/__init__.py index 4ec6d61b..42590288 100644 --- a/pipenv/patched/piptools/_compat/__init__.py +++ b/pipenv/patched/piptools/_compat/__init__.py @@ -11,3 +11,21 @@ if six.PY2: else: from tempfile import TemporaryDirectory from contextlib import ExitStack + +from .pip_compat import ( + InstallRequirement, + parse_requirements, + RequirementSet, + user_cache_dir, + FAVORITE_HASH, + is_file_url, + url_to_path, + PackageFinder, + FormatControl, + Wheel, + Command, + cmdoptions, + get_installed_distributions, + PyPI, + SafeFileCache, +) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py new file mode 100644 index 00000000..2b5c12a2 --- /dev/null +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -0,0 +1,40 @@ +# -*- coding=utf-8 -*- +import importlib + + +def do_import(module_path, subimport=None, old_path=None, vendored_name=None): + internal = 'pip._internal.{0}'.format(module_path) + old_path = old_path or module_path + pip9 = 'pip.{0}'.format(old_path) + _tmp = None + if vendored_name: + vendor = '{0}.{1}'.format(vendored_name, old_path) + try: + _tmp = importlib.import_module(vendor) + except ImportError: + pass + if not _tmp: + try: + _tmp = importlib.import_module(internal) + except ImportError: + _tmp = importlib.import_module(pip9) + if subimport: + return getattr(_tmp, subimport, _tmp) + return _tmp + + +InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name='notpip') +parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name='notpip') +RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name='notpip') +user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name='notpip') +FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name='notpip') +is_file_url = do_import('download', 'is_file_url', vendored_name='notpip') +url_to_path = do_import('download', 'url_to_path', vendored_name='notpip') +PackageFinder = do_import('index', 'PackageFinder', vendored_name='notpip') +FormatControl = do_import('index', 'FormatControl') +Wheel = do_import('wheel', 'Wheel', vendored_name='notpip') +Command = do_import('basecommand', 'Command') +cmdoptions = do_import('cmdoptions') +get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') +PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') +SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') diff --git a/pipenv/patched/piptools/cache.py b/pipenv/patched/piptools/cache.py index 5a44953b..7595b964 100644 --- a/pipenv/patched/piptools/cache.py +++ b/pipenv/patched/piptools/cache.py @@ -6,7 +6,7 @@ import json import os import sys -from pip9._vendor.packaging.requirements import Requirement +from pip._vendor.packaging.requirements import Requirement from .exceptions import PipToolsError from .locations import CACHE_DIR diff --git a/pipenv/patched/piptools/exceptions.py b/pipenv/patched/piptools/exceptions.py index 66df316f..77c5bd40 100644 --- a/pipenv/patched/piptools/exceptions.py +++ b/pipenv/patched/piptools/exceptions.py @@ -3,22 +3,44 @@ class PipToolsError(Exception): class NoCandidateFound(PipToolsError): - def __init__(self, ireq, candidates_tried, index_urls): + def __init__(self, ireq, candidates_tried, finder): self.ireq = ireq self.candidates_tried = candidates_tried - self.index_urls = index_urls + self.finder = finder def __str__(self): - sorted_versions = sorted(c.version for c in self.candidates_tried) + versions = [] + pre_versions = [] + + for candidate in sorted(self.candidates_tried): + version = str(candidate.version) + if candidate.version.is_prerelease: + pre_versions.append(version) + else: + versions.append(version) + lines = [ 'Could not find a version that matches {}'.format(self.ireq), - 'Tried: {}'.format(', '.join(str(version) for version in sorted_versions) or '(no version found at all)') ] - if sorted_versions: + + if versions: + lines.append('Tried: {}'.format(', '.join(versions))) + + if pre_versions: + if self.finder.allow_all_prereleases: + line = 'Tried' + else: + line = 'Skipped' + + line += ' pre-versions: {}'.format(', '.join(pre_versions)) + lines.append(line) + + if versions or pre_versions: lines.append('There are incompatible versions in the resolved dependencies.') else: + lines.append('No versions found') lines.append('{} {} reachable?'.format( - 'Were' if len(self.index_urls) > 1 else 'Was', ' or '.join(self.index_urls)) + 'Were' if len(self.finder.index_urls) > 1 else 'Was', ' or '.join(self.finder.index_urls)) ) return '\n'.join(lines) diff --git a/pipenv/patched/piptools/locations.py b/pipenv/patched/piptools/locations.py index 5791f0f9..0c43cb8c 100644 --- a/pipenv/patched/piptools/locations.py +++ b/pipenv/patched/piptools/locations.py @@ -3,7 +3,7 @@ from shutil import rmtree from .click import secho # Patch by vphilippon 2017-11-22: Use pipenv cache path. -# from pip9.utils.appdirs import user_cache_dir +# from ._compat import user_cache_dir from pipenv.environments import PIPENV_CACHE_DIR # The user_cache_dir helper comes straight from pip itself diff --git a/pipenv/patched/piptools/repositories/base.py b/pipenv/patched/piptools/repositories/base.py index 69835c07..57e85fda 100644 --- a/pipenv/patched/piptools/repositories/base.py +++ b/pipenv/patched/piptools/repositories/base.py @@ -44,5 +44,5 @@ class BaseRepository(object): @contextmanager def allow_all_wheels(self): """ - Monkey patches pip9.Wheel to allow wheels from all platforms and Python versions. + Monkey patches pip.Wheel to allow wheels from all platforms and Python versions. """ diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py index 3c6182b7..08dabe12 100644 --- a/pipenv/patched/piptools/repositories/local.py +++ b/pipenv/patched/piptools/repositories/local.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from piptools.utils import as_tuple, key_from_req, make_install_requirement from .base import BaseRepository -from pip9.utils.hashes import FAVORITE_HASH +from .._compat import FAVORITE_HASH def ireq_satisfied_by_existing_pin(ireq, existing_pin): diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index e5031979..a2607446 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -7,17 +7,22 @@ import os from contextlib import contextmanager from shutil import rmtree -from notpip.download import is_file_url, url_to_path -from notpip.index import PackageFinder -from notpip.req.req_set import RequirementSet -from notpip.wheel import Wheel -from notpip.req.req_install import InstallRequirement +from .._compat import ( + is_file_url, + url_to_path, + PackageFinder, + RequirementSet, + Wheel, + FAVORITE_HASH, + TemporaryDirectory, + PyPI, + InstallRequirement, + SafeFileCache, +) + from pip9._vendor.packaging.requirements import InvalidRequirement from pip9._vendor.pyparsing import ParseException -from notpip.download import SafeFileCache -from notpip.utils.hashes import FAVORITE_HASH -from .._compat import TemporaryDirectory from ..cache import CACHE_DIR from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound @@ -26,11 +31,23 @@ from ..utils import (fs_str, is_pinned_requirement, lookup_table, from .base import BaseRepository +try: + from pip._internal.operations.prepare import RequirementPreparer + from pip._internal.resolve import Resolver as PipResolver +except ImportError: + pass + +try: + from pip._internal.cache import WheelCache +except ImportError: + from pip.wheel import WheelCache + + class HashCache(SafeFileCache): """Caches hashes of PyPI artifacts so we do not need to re-download them - Hashes are only cached when the URL appears to contain a hash in it (and the cache key includes - the hash value returned from the server). This ought to avoid issues where the location on the + Hashes are only cached when the URL appears to contain a hash in it and the cache key includes + the hash value returned from the server). This ought to avoid ssues where the location on the server changes.""" def __init__(self, *args, **kwargs): session = kwargs.pop('session') @@ -39,7 +56,7 @@ class HashCache(SafeFileCache): super(HashCache, self).__init__(*args, **kwargs) def get_hash(self, location): - # if there is no location hash (i.e., md5 / sha256 / etc) we don't want to store it + # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None can_hash = location.hash if can_hash: @@ -61,7 +78,7 @@ class HashCache(SafeFileCache): class PyPIRepository(BaseRepository): - DEFAULT_INDEX_URL = 'https://pypi.org/simple' + DEFAULT_INDEX_URL = PyPI.simple_url """ The PyPIRepository will use the provided Finder instance to lookup @@ -72,6 +89,8 @@ class PyPIRepository(BaseRepository): def __init__(self, pip_options, session, use_json=False): self.session = session self.use_json = use_json + self.pip_options = pip_options + self.wheel_cache = WheelCache(CACHE_DIR, pip_options.format_control) index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: @@ -148,7 +167,7 @@ class PyPIRepository(BaseRepository): # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] if not matching_candidates: - raise NoCandidateFound(ireq, all_candidates, self.finder.index_urls) + raise NoCandidateFound(ireq, all_candidates, self.finder) best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -174,8 +193,9 @@ class PyPIRepository(BaseRepository): # TODO: Latest isn't always latest. latest = list(r.json()['releases'].keys())[-1] if str(ireq.req.specifier) == '=={0}'.format(latest): - - for requires in r.json().get('info', {}).get('requires_dist', {}): + latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest) + latest_requires = self.session.get(latest_url) + for requires in latest_requires.json().get('info', {}).get('requires_dist', {}): i = InstallRequirement.from_line(requires) if 'extra' not in repr(i.markers): @@ -189,7 +209,6 @@ class PyPIRepository(BaseRepository): except Exception: return set() - def get_dependencies(self, ireq): json_results = set() @@ -226,7 +245,6 @@ class PyPIRepository(BaseRepository): except TypeError: pass - if ireq not in self._dependencies_cache: if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # No download_dir for locally available editable requirements. @@ -244,17 +262,52 @@ class PyPIRepository(BaseRepository): if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) - reqset = RequirementSet(self.build_dir, - self.source_dir, - download_dir=download_dir, - wheel_download_dir=self._wheel_download_dir, - session=self.session, - ignore_installed=True, - ignore_compatibility=False - ) - - result = reqset._prepare_file(self.finder, ireq, ignore_requires_python=True) - + try: + # Pip < 9 and below + reqset = RequirementSet( + self.build_dir, + self.source_dir, + download_dir=download_dir, + wheel_download_dir=self._wheel_download_dir, + session=self.session, + ignore_installed=True, + ignore_compatibility=False, + wheel_cache=self.wheel_cache, + ) + result = reqset._prepare_file( + self.finder, + ireq, + ignore_requires_python=True + ) + except TypeError: + # Pip >= 10 (new resolver!) + preparer = RequirementPreparer( + build_dir=self.build_dir, + src_dir=self.source_dir, + download_dir=download_dir, + wheel_download_dir=self._wheel_download_dir, + progress_bar='off', + build_isolation=False + ) + reqset = RequirementSet() + ireq.is_direct = True + reqset.add_requirement(ireq) + self.resolver = PipResolver( + preparer=preparer, + finder=self.finder, + session=self.session, + upgrade_strategy="to-satisfy-only", + force_reinstall=False, + ignore_dependencies=False, + ignore_requires_python=False, + ignore_installed=True, + isolated=False, + wheel_cache=self.wheel_cache, + use_user_site=False, + ignore_compatibility=False + ) + self.resolver.resolve(reqset) + result = reqset.requirements.values() # Convert setup_requires dict into a somewhat usable form. if setup_requires: for section in setup_requires: @@ -279,12 +332,12 @@ class PyPIRepository(BaseRepository): pass if reqset.requires_python: - marker = 'python_version=="{0}"'.format(reqset.requires_python.replace(' ', '')) new_req = InstallRequirement.from_line('{0}; {1}'.format(str(ireq.req), marker)) result = [new_req] self._dependencies_cache[ireq] = result + reqset.cleanup_files() return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): @@ -317,7 +370,7 @@ class PyPIRepository(BaseRepository): @contextmanager def allow_all_wheels(self): """ - Monkey patches pip9.Wheel to allow wheels from all platforms and Python versions. + Monkey patches pip.Wheel to allow wheels from all platforms and Python versions. This also saves the candidate cache and set a new one, or else the results from the previous non-patched calls will interfere. @@ -351,7 +404,7 @@ def open_local_or_remote_file(link, session): """ Open local or remote file for reading. - :type link: pip9.index.Link + :type link: pip.index.Link :type session: requests.Session :raises ValueError: If link points to a local directory. :return: a context manager to the opened file-like object diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 64b11997..6380e8f6 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -8,7 +8,7 @@ from itertools import chain, count import os from first import first -from pip9.req import InstallRequirement +from ._compat import InstallRequirement from . import click from .cache import DependencyCache @@ -302,7 +302,7 @@ class Resolver(object): dependency_strings = self.dependency_cache[ireq] log.debug(' {:25} requires {}'.format(format_requirement(ireq), ', '.join(sorted(dependency_strings, key=lambda s: s.lower())) or '-')) - from notpip._vendor.packaging.markers import InvalidMarker + from pip9._vendor.packaging.markers import InvalidMarker for dependency_string in dependency_strings: try: _dependency_string = dependency_string @@ -315,7 +315,6 @@ class Resolver(object): except InvalidMarker: yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) - def reverse_dependencies(self, ireqs): non_editable = [ireq for ireq in ireqs if not ireq.editable] return self.dependency_cache.reverse_dependencies(non_editable) diff --git a/pipenv/patched/piptools/scripts/compile.py b/pipenv/patched/piptools/scripts/compile.py index 8194fcf9..484b710c 100644 --- a/pipenv/patched/piptools/scripts/compile.py +++ b/pipenv/patched/piptools/scripts/compile.py @@ -7,8 +7,12 @@ import os import sys import tempfile -import pip9 -from pip9.req import InstallRequirement, parse_requirements +from .._compat import ( + InstallRequirement, + parse_requirements, + cmdoptions, + Command, +) from .. import click from ..exceptions import PipToolsError @@ -21,7 +25,7 @@ from ..writer import OutputWriter DEFAULT_REQUIREMENTS_FILE = 'requirements.in' -class PipCommand(pip9.basecommand.Command): +class PipCommand(Command): name = 'PipCommand' @@ -247,12 +251,14 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, def get_pip_command(): - # Use pip's parser for pip9.conf management and defaults. + # Use pip's parser for pip.conf management and defaults. # General options (find_links, index_url, extra_index_url, trusted_host, - # and pre) are defered to pip9. + # and pre) are defered to pip. pip_command = PipCommand() - index_opts = pip9.cmdoptions.make_option_group( - pip9.cmdoptions.index_group, + pip_command.parser.add_option(cmdoptions.no_binary()) + pip_command.parser.add_option(cmdoptions.only_binary()) + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, pip_command.parser, ) pip_command.parser.insert_option_group(0, index_opts) diff --git a/pipenv/patched/piptools/scripts/sync.py b/pipenv/patched/piptools/scripts/sync.py index c43c5af8..610c1d5e 100644 --- a/pipenv/patched/piptools/scripts/sync.py +++ b/pipenv/patched/piptools/scripts/sync.py @@ -5,9 +5,9 @@ from __future__ import (absolute_import, division, print_function, import os import sys -import pip9 from .. import click, sync +from .._compat import parse_requirements, get_installed_distributions from ..exceptions import PipToolsError from ..logging import log from ..utils import flat_map @@ -45,7 +45,7 @@ def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet, log.error('ERROR: ' + msg) sys.exit(2) - requirements = flat_map(lambda src: pip9.req.parse_requirements(src, session=True), + requirements = flat_map(lambda src: parse_requirements(src, session=True), src_files) try: @@ -54,7 +54,7 @@ def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet, log.error(str(e)) sys.exit(2) - installed_dists = pip9.get_installed_distributions(skip=[], user_only=user_only) + installed_dists = get_installed_distributions(skip=[], user_only=user_only) to_install, to_uninstall = sync.diff(requirements, installed_dists) install_flags = [] diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index db8bb9b3..d76695d4 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -2,11 +2,13 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import os import sys from itertools import chain, groupby from collections import OrderedDict +from contextlib import contextmanager -from pip9.req import InstallRequirement +from ._compat import InstallRequirement from first import first @@ -240,3 +242,17 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() + + +# Borrowed from pew to avoid importing pew which imports psutil +# See https://github.com/berdario/pew/blob/master/pew/_utils.py#L82 +@contextmanager +def temp_environ(): + """Allow the ability to set os.environ temporarily""" + environ = dict(os.environ) + try: + yield + + finally: + os.environ.clear() + os.environ.update(environ) diff --git a/pipenv/patched/safety/cli.py b/pipenv/patched/safety/cli.py index fc543ef5..37ae3acb 100644 --- a/pipenv/patched/safety/cli.py +++ b/pipenv/patched/safety/cli.py @@ -15,7 +15,7 @@ try: from pip import get_installed_distributions except ImportError: # pip 10 - from pip9._internal.utils.misc import get_installed_distributions + from pip._internal.utils.misc import get_installed_distributions @click.group() From 8abb764ce866d3a5195bec7f0d3c381db5569516 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 17 May 2018 14:41:24 -0400 Subject: [PATCH 6/7] Update pypi urls Signed-off-by: Dan Ryan --- tests/integration/test_install_markers.py | 2 +- tests/integration/test_install_uri.py | 2 +- tests/integration/test_lock.py | 6 +++--- tests/integration/test_project.py | 6 +++--- tests/pytest-pypi/Pipfile | 2 +- tests/pytest-pypi/Pipfile.lock | 2 +- tests/unit/test_utils.py | 16 ++++++++-------- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index 87235d4a..e2f55415 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -132,7 +132,7 @@ def test_environment_variable_value_does_not_change_hash(PipenvInstance, pypi): with open(p.pipfile_path, 'w') as f: f.write(""" [[source]] -url = 'https://${PYPI_USERNAME}:${PYPI_PASSWORD}@pypi.python.org/simple' +url = 'https://${PYPI_USERNAME}:${PYPI_PASSWORD}@pypi.org/simple' verify_ssl = true name = 'pypi' [requires] diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 94391f57..646ca578 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -103,7 +103,7 @@ def test_install_named_index_alias(PipenvInstance, pypi): with open(p.pipfile_path, 'w') as f: contents = """ [[source]] -url = "https://pypi.python.org/simple" +url = "https://pypi.org/simple" verify_ssl = true name = "pypi" diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index a14b8b10..45206d76 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -199,7 +199,7 @@ def test_private_index_skip_lock(PipenvInstance): with open(p.pipfile_path, 'w') as f: contents = """ [[source]] -url = "https://pypi.python.org/simple" +url = "https://pypi.org/simple" verify_ssl = true name = "pypi" @@ -228,7 +228,7 @@ def test_private_index_lock_requirements(PipenvInstance): with open(p.pipfile_path, 'w') as f: contents = """ [[source]] -url = "https://pypi.python.org/simple" +url = "https://pypi.org/simple" verify_ssl = true name = "pypi" @@ -246,7 +246,7 @@ requests = "*" assert c.return_code == 0 c = p.pipenv('lock -r') assert c.return_code == 0 - assert '-i https://pypi.python.org/simple' in c.out.strip() + assert '-i https://pypi.org/simple' in c.out.strip() assert '--extra-index-url https://test.pypi.org/simple' in c.out.strip() diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 9a03c5c1..3419d4c7 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -42,7 +42,7 @@ verify_ssl = false name = "testindex" [[source]] -url = "https://pypi.python.org/simple" +url = "https://pypi.org/simple" verify_ssl = "true" name = "pypi" @@ -60,7 +60,7 @@ six = {{version = "*", index = "pypi"}} assert c.return_code == 0 project = Project() sources = [ - ['pypi', 'https://pypi.python.org/simple'], + ['pypi', 'https://pypi.org/simple'], ['testindex', os.environ.get('PIPENV_TEST_INDEX')] ] for src in sources: @@ -130,7 +130,7 @@ verify_ssl = "true" name = "pypi" [[source]] -url = "https://pypi.python.org/simple" +url = "https://pypi.org/simple" verify_ssl = "true" name = "legacy" diff --git a/tests/pytest-pypi/Pipfile b/tests/pytest-pypi/Pipfile index 366b2617..6abe710c 100644 --- a/tests/pytest-pypi/Pipfile +++ b/tests/pytest-pypi/Pipfile @@ -1,6 +1,6 @@ [[source]] -url = "https://pypi.python.org/simple" +url = "https://pypi.org/simple" verify_ssl = true name = "pypi" diff --git a/tests/pytest-pypi/Pipfile.lock b/tests/pytest-pypi/Pipfile.lock index b60f1990..25978b9d 100644 --- a/tests/pytest-pypi/Pipfile.lock +++ b/tests/pytest-pypi/Pipfile.lock @@ -10,7 +10,7 @@ "sources": [ { "name": "pypi", - "url": "https://pypi.python.org/simple", + "url": "https://pypi.org/simple", "verify_ssl": true } ] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index f314b7e9..82bdc2f1 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -389,26 +389,26 @@ twine = "*" ([{'url': 'https://test.example.com/simple', 'verify_ssl': False}], ['-i', 'https://test.example.com/simple', '--trusted-host', 'test.example.com']), - ([{'url': "https://pypi.python.org/simple"}, + ([{'url': "https://pypi.org/simple"}, {'url': "https://custom.example.com/simple"}], - ['-i', 'https://pypi.python.org/simple', + ['-i', 'https://pypi.org/simple', '--extra-index-url', 'https://custom.example.com/simple']), - ([{'url': "https://pypi.python.org/simple"}, + ([{'url': "https://pypi.org/simple"}, {'url': "https://custom.example.com/simple", 'verify_ssl': False}], - ['-i', 'https://pypi.python.org/simple', + ['-i', 'https://pypi.org/simple', '--extra-index-url', 'https://custom.example.com/simple', '--trusted-host', 'custom.example.com']), - ([{'url': "https://pypi.python.org/simple"}, + ([{'url': "https://pypi.org/simple"}, {'url': "https://user:password@custom.example.com/simple", 'verify_ssl': False}], - ['-i', 'https://pypi.python.org/simple', + ['-i', 'https://pypi.org/simple', '--extra-index-url', 'https://user:password@custom.example.com/simple', '--trusted-host', 'custom.example.com']), - ([{'url': "https://pypi.python.org/simple"}, + ([{'url': "https://pypi.org/simple"}, {'url': "https://user:password@custom.example.com/simple",}], - ['-i', 'https://pypi.python.org/simple', + ['-i', 'https://pypi.org/simple', '--extra-index-url', 'https://user:password@custom.example.com/simple',]), ], ) From 7227c9b2c925564f4c5397cd11d736831d71eae3 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 17 May 2018 18:51:53 -0400 Subject: [PATCH 7/7] Fix pip compat imports for piptools Signed-off-by: Dan Ryan --- pipenv/patched/piptools/_compat/pip_compat.py | 8 ++-- pipenv/patched/piptools/repositories/pypi.py | 6 +-- .../vendoring/patches/patched/piptools.patch | 48 ++++++++++++------- tests/integration/test_project.py | 2 +- 4 files changed, 39 insertions(+), 25 deletions(-) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py index 2b5c12a2..96c8a1ea 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -31,10 +31,10 @@ FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name='notpip is_file_url = do_import('download', 'is_file_url', vendored_name='notpip') url_to_path = do_import('download', 'url_to_path', vendored_name='notpip') PackageFinder = do_import('index', 'PackageFinder', vendored_name='notpip') -FormatControl = do_import('index', 'FormatControl') +FormatControl = do_import('index', 'FormatControl', vendored_name='notpip') Wheel = do_import('wheel', 'Wheel', vendored_name='notpip') -Command = do_import('basecommand', 'Command') -cmdoptions = do_import('cmdoptions') -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') +Command = do_import('basecommand', 'Command', vendored_name='pip9') +cmdoptions = do_import('cmdoptions', vendored_name='pip9') +get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name='pip9') PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index a2607446..1c808f36 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -90,7 +90,7 @@ class PyPIRepository(BaseRepository): self.session = session self.use_json = use_json self.pip_options = pip_options - self.wheel_cache = WheelCache(CACHE_DIR, pip_options.format_control) + self.wheel_cache = WheelCache(PIPENV_CACHE_DIR, pip_options.format_control) index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: @@ -122,8 +122,8 @@ class PyPIRepository(BaseRepository): # Setup file paths self.freshen_build_caches() - self._download_dir = fs_str(os.path.join(CACHE_DIR, 'pkgs')) - self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, 'wheels')) + self._download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'pkgs')) + self._wheel_download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'wheels')) def freshen_build_caches(self): """ diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 6f14002e..678a1bfd 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -18,10 +18,10 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the -diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..39a285b 100644 ---- a/pipenv/patched/piptools/repositories/pypi.py -+++ b/pipenv/patched/piptools/repositories/pypi.py +diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py +index 1c4b943..1c808f3 100644 +--- a/piptools/repositories/pypi.py ++++ b/piptools/repositories/pypi.py @@ -15,10 +15,16 @@ from .._compat import ( Wheel, FAVORITE_HASH, @@ -81,7 +81,7 @@ index 1c4b943..39a285b 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,8 +86,9 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +86,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -90,9 +90,12 @@ index 1c4b943..39a285b 100644 self.session = session + self.use_json = use_json self.pip_options = pip_options - self.wheel_cache = WheelCache(CACHE_DIR, pip_options.format_control) +- self.wheel_cache = WheelCache(CACHE_DIR, pip_options.format_control) ++ self.wheel_cache = WheelCache(PIPENV_CACHE_DIR, pip_options.format_control) -@@ -74,6 +115,10 @@ class PyPIRepository(BaseRepository): + index_urls = [pip_options.index_url] + pip_options.extra_index_urls + if pip_options.no_index: +@@ -74,11 +115,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -103,6 +106,13 @@ index 1c4b943..39a285b 100644 # Setup file paths self.freshen_build_caches() +- self._download_dir = fs_str(os.path.join(CACHE_DIR, 'pkgs')) +- self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, 'wheels')) ++ self._download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'pkgs')) ++ self._wheel_download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'wheels')) + + def freshen_build_caches(self): + """ @@ -126,11 +171,60 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) @@ -394,10 +404,10 @@ index fde5816..d76695d 100644 return line -diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py -index 7e8cdf3..2b5c12a 100644 ---- a/pipenv/patched/piptools/_compat/pip_compat.py -+++ b/pipenv/patched/piptools/_compat/pip_compat.py +diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py +index 7e8cdf3..96c8a1e 100644 +--- a/piptools/_compat/pip_compat.py ++++ b/piptools/_compat/pip_compat.py @@ -1,30 +1,40 @@ # -*- coding=utf-8 -*- import importlib @@ -437,6 +447,12 @@ index 7e8cdf3..2b5c12a 100644 -is_file_url = do_import('download', 'is_file_url') -url_to_path = do_import('download', 'url_to_path') -PackageFinder = do_import('index', 'PackageFinder') +-FormatControl = do_import('index', 'FormatControl') +-Wheel = do_import('wheel', 'Wheel') +-Command = do_import('basecommand', 'Command') +-cmdoptions = do_import('cmdoptions') +-get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') +-PyPI = do_import('models.index', 'PyPI') +InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name='notpip') +parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name='notpip') +RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name='notpip') @@ -445,13 +461,11 @@ index 7e8cdf3..2b5c12a 100644 +is_file_url = do_import('download', 'is_file_url', vendored_name='notpip') +url_to_path = do_import('download', 'url_to_path', vendored_name='notpip') +PackageFinder = do_import('index', 'PackageFinder', vendored_name='notpip') - FormatControl = do_import('index', 'FormatControl') --Wheel = do_import('wheel', 'Wheel') ++FormatControl = do_import('index', 'FormatControl', vendored_name='notpip') +Wheel = do_import('wheel', 'Wheel', vendored_name='notpip') - Command = do_import('basecommand', 'Command') - cmdoptions = do_import('cmdoptions') - get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') --PyPI = do_import('models.index', 'PyPI') ++Command = do_import('basecommand', 'Command', vendored_name='pip9') ++cmdoptions = do_import('cmdoptions', vendored_name='pip9') ++get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name='pip9') +PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') +SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') diff --git a/pipenv/patched/piptools/_compat/__init__.py b/piptools/_compat/__init__.py diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 3419d4c7..5deccc84 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -130,7 +130,7 @@ verify_ssl = "true" name = "pypi" [[source]] -url = "https://pypi.org/simple" +url = "https://pypi.python.org/simple" verify_ssl = "true" name = "legacy"