From 7151583f1ca69e55b82c48ff6ca16e3025fa2b0c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 17 May 2018 14:29:07 -0400 Subject: [PATCH] Revendor piptools for update Signed-off-by: Dan Ryan --- pipenv/patched/piptools/LICENSE.txt | 17 +++ pipenv/patched/piptools/__init__.py | 6 - pipenv/patched/piptools/_compat/__init__.py | 18 +++ pipenv/patched/piptools/_compat/pip_compat.py | 40 ++++++ pipenv/patched/piptools/cache.py | 2 +- pipenv/patched/piptools/exceptions.py | 34 +++++- pipenv/patched/piptools/locations.py | 2 +- pipenv/patched/piptools/repositories/base.py | 2 +- pipenv/patched/piptools/repositories/local.py | 2 +- pipenv/patched/piptools/repositories/pypi.py | 115 +++++++++++++----- pipenv/patched/piptools/resolver.py | 5 +- pipenv/patched/piptools/scripts/compile.py | 20 +-- pipenv/patched/piptools/scripts/sync.py | 6 +- pipenv/patched/piptools/utils.py | 18 ++- pipenv/patched/safety/cli.py | 2 +- 15 files changed, 227 insertions(+), 62 deletions(-) create mode 100644 pipenv/patched/piptools/LICENSE.txt create mode 100644 pipenv/patched/piptools/_compat/pip_compat.py diff --git a/pipenv/patched/piptools/LICENSE.txt b/pipenv/patched/piptools/LICENSE.txt new file mode 100644 index 00000000..89de3547 --- /dev/null +++ b/pipenv/patched/piptools/LICENSE.txt @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/patched/piptools/__init__.py b/pipenv/patched/piptools/__init__.py index f332ac15..e69de29b 100644 --- a/pipenv/patched/piptools/__init__.py +++ b/pipenv/patched/piptools/__init__.py @@ -1,6 +0,0 @@ -import os -import sys - -# Inject vendored directory into system path. -v_path = os.path.abspath(os.path.sep.join([os.path.dirname(os.path.realpath(__file__)), '_vendored'])) -sys.path.insert(0, v_path) diff --git a/pipenv/patched/piptools/_compat/__init__.py b/pipenv/patched/piptools/_compat/__init__.py index 4ec6d61b..42590288 100644 --- a/pipenv/patched/piptools/_compat/__init__.py +++ b/pipenv/patched/piptools/_compat/__init__.py @@ -11,3 +11,21 @@ if six.PY2: else: from tempfile import TemporaryDirectory from contextlib import ExitStack + +from .pip_compat import ( + InstallRequirement, + parse_requirements, + RequirementSet, + user_cache_dir, + FAVORITE_HASH, + is_file_url, + url_to_path, + PackageFinder, + FormatControl, + Wheel, + Command, + cmdoptions, + get_installed_distributions, + PyPI, + SafeFileCache, +) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py new file mode 100644 index 00000000..2b5c12a2 --- /dev/null +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -0,0 +1,40 @@ +# -*- coding=utf-8 -*- +import importlib + + +def do_import(module_path, subimport=None, old_path=None, vendored_name=None): + internal = 'pip._internal.{0}'.format(module_path) + old_path = old_path or module_path + pip9 = 'pip.{0}'.format(old_path) + _tmp = None + if vendored_name: + vendor = '{0}.{1}'.format(vendored_name, old_path) + try: + _tmp = importlib.import_module(vendor) + except ImportError: + pass + if not _tmp: + try: + _tmp = importlib.import_module(internal) + except ImportError: + _tmp = importlib.import_module(pip9) + if subimport: + return getattr(_tmp, subimport, _tmp) + return _tmp + + +InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name='notpip') +parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name='notpip') +RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name='notpip') +user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name='notpip') +FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name='notpip') +is_file_url = do_import('download', 'is_file_url', vendored_name='notpip') +url_to_path = do_import('download', 'url_to_path', vendored_name='notpip') +PackageFinder = do_import('index', 'PackageFinder', vendored_name='notpip') +FormatControl = do_import('index', 'FormatControl') +Wheel = do_import('wheel', 'Wheel', vendored_name='notpip') +Command = do_import('basecommand', 'Command') +cmdoptions = do_import('cmdoptions') +get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') +PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') +SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') diff --git a/pipenv/patched/piptools/cache.py b/pipenv/patched/piptools/cache.py index 5a44953b..7595b964 100644 --- a/pipenv/patched/piptools/cache.py +++ b/pipenv/patched/piptools/cache.py @@ -6,7 +6,7 @@ import json import os import sys -from pip9._vendor.packaging.requirements import Requirement +from pip._vendor.packaging.requirements import Requirement from .exceptions import PipToolsError from .locations import CACHE_DIR diff --git a/pipenv/patched/piptools/exceptions.py b/pipenv/patched/piptools/exceptions.py index 66df316f..77c5bd40 100644 --- a/pipenv/patched/piptools/exceptions.py +++ b/pipenv/patched/piptools/exceptions.py @@ -3,22 +3,44 @@ class PipToolsError(Exception): class NoCandidateFound(PipToolsError): - def __init__(self, ireq, candidates_tried, index_urls): + def __init__(self, ireq, candidates_tried, finder): self.ireq = ireq self.candidates_tried = candidates_tried - self.index_urls = index_urls + self.finder = finder def __str__(self): - sorted_versions = sorted(c.version for c in self.candidates_tried) + versions = [] + pre_versions = [] + + for candidate in sorted(self.candidates_tried): + version = str(candidate.version) + if candidate.version.is_prerelease: + pre_versions.append(version) + else: + versions.append(version) + lines = [ 'Could not find a version that matches {}'.format(self.ireq), - 'Tried: {}'.format(', '.join(str(version) for version in sorted_versions) or '(no version found at all)') ] - if sorted_versions: + + if versions: + lines.append('Tried: {}'.format(', '.join(versions))) + + if pre_versions: + if self.finder.allow_all_prereleases: + line = 'Tried' + else: + line = 'Skipped' + + line += ' pre-versions: {}'.format(', '.join(pre_versions)) + lines.append(line) + + if versions or pre_versions: lines.append('There are incompatible versions in the resolved dependencies.') else: + lines.append('No versions found') lines.append('{} {} reachable?'.format( - 'Were' if len(self.index_urls) > 1 else 'Was', ' or '.join(self.index_urls)) + 'Were' if len(self.finder.index_urls) > 1 else 'Was', ' or '.join(self.finder.index_urls)) ) return '\n'.join(lines) diff --git a/pipenv/patched/piptools/locations.py b/pipenv/patched/piptools/locations.py index 5791f0f9..0c43cb8c 100644 --- a/pipenv/patched/piptools/locations.py +++ b/pipenv/patched/piptools/locations.py @@ -3,7 +3,7 @@ from shutil import rmtree from .click import secho # Patch by vphilippon 2017-11-22: Use pipenv cache path. -# from pip9.utils.appdirs import user_cache_dir +# from ._compat import user_cache_dir from pipenv.environments import PIPENV_CACHE_DIR # The user_cache_dir helper comes straight from pip itself diff --git a/pipenv/patched/piptools/repositories/base.py b/pipenv/patched/piptools/repositories/base.py index 69835c07..57e85fda 100644 --- a/pipenv/patched/piptools/repositories/base.py +++ b/pipenv/patched/piptools/repositories/base.py @@ -44,5 +44,5 @@ class BaseRepository(object): @contextmanager def allow_all_wheels(self): """ - Monkey patches pip9.Wheel to allow wheels from all platforms and Python versions. + Monkey patches pip.Wheel to allow wheels from all platforms and Python versions. """ diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py index 3c6182b7..08dabe12 100644 --- a/pipenv/patched/piptools/repositories/local.py +++ b/pipenv/patched/piptools/repositories/local.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from piptools.utils import as_tuple, key_from_req, make_install_requirement from .base import BaseRepository -from pip9.utils.hashes import FAVORITE_HASH +from .._compat import FAVORITE_HASH def ireq_satisfied_by_existing_pin(ireq, existing_pin): diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index e5031979..a2607446 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -7,17 +7,22 @@ import os from contextlib import contextmanager from shutil import rmtree -from notpip.download import is_file_url, url_to_path -from notpip.index import PackageFinder -from notpip.req.req_set import RequirementSet -from notpip.wheel import Wheel -from notpip.req.req_install import InstallRequirement +from .._compat import ( + is_file_url, + url_to_path, + PackageFinder, + RequirementSet, + Wheel, + FAVORITE_HASH, + TemporaryDirectory, + PyPI, + InstallRequirement, + SafeFileCache, +) + from pip9._vendor.packaging.requirements import InvalidRequirement from pip9._vendor.pyparsing import ParseException -from notpip.download import SafeFileCache -from notpip.utils.hashes import FAVORITE_HASH -from .._compat import TemporaryDirectory from ..cache import CACHE_DIR from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound @@ -26,11 +31,23 @@ from ..utils import (fs_str, is_pinned_requirement, lookup_table, from .base import BaseRepository +try: + from pip._internal.operations.prepare import RequirementPreparer + from pip._internal.resolve import Resolver as PipResolver +except ImportError: + pass + +try: + from pip._internal.cache import WheelCache +except ImportError: + from pip.wheel import WheelCache + + class HashCache(SafeFileCache): """Caches hashes of PyPI artifacts so we do not need to re-download them - Hashes are only cached when the URL appears to contain a hash in it (and the cache key includes - the hash value returned from the server). This ought to avoid issues where the location on the + Hashes are only cached when the URL appears to contain a hash in it and the cache key includes + the hash value returned from the server). This ought to avoid ssues where the location on the server changes.""" def __init__(self, *args, **kwargs): session = kwargs.pop('session') @@ -39,7 +56,7 @@ class HashCache(SafeFileCache): super(HashCache, self).__init__(*args, **kwargs) def get_hash(self, location): - # if there is no location hash (i.e., md5 / sha256 / etc) we don't want to store it + # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None can_hash = location.hash if can_hash: @@ -61,7 +78,7 @@ class HashCache(SafeFileCache): class PyPIRepository(BaseRepository): - DEFAULT_INDEX_URL = 'https://pypi.org/simple' + DEFAULT_INDEX_URL = PyPI.simple_url """ The PyPIRepository will use the provided Finder instance to lookup @@ -72,6 +89,8 @@ class PyPIRepository(BaseRepository): def __init__(self, pip_options, session, use_json=False): self.session = session self.use_json = use_json + self.pip_options = pip_options + self.wheel_cache = WheelCache(CACHE_DIR, pip_options.format_control) index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: @@ -148,7 +167,7 @@ class PyPIRepository(BaseRepository): # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] if not matching_candidates: - raise NoCandidateFound(ireq, all_candidates, self.finder.index_urls) + raise NoCandidateFound(ireq, all_candidates, self.finder) best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -174,8 +193,9 @@ class PyPIRepository(BaseRepository): # TODO: Latest isn't always latest. latest = list(r.json()['releases'].keys())[-1] if str(ireq.req.specifier) == '=={0}'.format(latest): - - for requires in r.json().get('info', {}).get('requires_dist', {}): + latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest) + latest_requires = self.session.get(latest_url) + for requires in latest_requires.json().get('info', {}).get('requires_dist', {}): i = InstallRequirement.from_line(requires) if 'extra' not in repr(i.markers): @@ -189,7 +209,6 @@ class PyPIRepository(BaseRepository): except Exception: return set() - def get_dependencies(self, ireq): json_results = set() @@ -226,7 +245,6 @@ class PyPIRepository(BaseRepository): except TypeError: pass - if ireq not in self._dependencies_cache: if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # No download_dir for locally available editable requirements. @@ -244,17 +262,52 @@ class PyPIRepository(BaseRepository): if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) - reqset = RequirementSet(self.build_dir, - self.source_dir, - download_dir=download_dir, - wheel_download_dir=self._wheel_download_dir, - session=self.session, - ignore_installed=True, - ignore_compatibility=False - ) - - result = reqset._prepare_file(self.finder, ireq, ignore_requires_python=True) - + try: + # Pip < 9 and below + reqset = RequirementSet( + self.build_dir, + self.source_dir, + download_dir=download_dir, + wheel_download_dir=self._wheel_download_dir, + session=self.session, + ignore_installed=True, + ignore_compatibility=False, + wheel_cache=self.wheel_cache, + ) + result = reqset._prepare_file( + self.finder, + ireq, + ignore_requires_python=True + ) + except TypeError: + # Pip >= 10 (new resolver!) + preparer = RequirementPreparer( + build_dir=self.build_dir, + src_dir=self.source_dir, + download_dir=download_dir, + wheel_download_dir=self._wheel_download_dir, + progress_bar='off', + build_isolation=False + ) + reqset = RequirementSet() + ireq.is_direct = True + reqset.add_requirement(ireq) + self.resolver = PipResolver( + preparer=preparer, + finder=self.finder, + session=self.session, + upgrade_strategy="to-satisfy-only", + force_reinstall=False, + ignore_dependencies=False, + ignore_requires_python=False, + ignore_installed=True, + isolated=False, + wheel_cache=self.wheel_cache, + use_user_site=False, + ignore_compatibility=False + ) + self.resolver.resolve(reqset) + result = reqset.requirements.values() # Convert setup_requires dict into a somewhat usable form. if setup_requires: for section in setup_requires: @@ -279,12 +332,12 @@ class PyPIRepository(BaseRepository): pass if reqset.requires_python: - marker = 'python_version=="{0}"'.format(reqset.requires_python.replace(' ', '')) new_req = InstallRequirement.from_line('{0}; {1}'.format(str(ireq.req), marker)) result = [new_req] self._dependencies_cache[ireq] = result + reqset.cleanup_files() return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): @@ -317,7 +370,7 @@ class PyPIRepository(BaseRepository): @contextmanager def allow_all_wheels(self): """ - Monkey patches pip9.Wheel to allow wheels from all platforms and Python versions. + Monkey patches pip.Wheel to allow wheels from all platforms and Python versions. This also saves the candidate cache and set a new one, or else the results from the previous non-patched calls will interfere. @@ -351,7 +404,7 @@ def open_local_or_remote_file(link, session): """ Open local or remote file for reading. - :type link: pip9.index.Link + :type link: pip.index.Link :type session: requests.Session :raises ValueError: If link points to a local directory. :return: a context manager to the opened file-like object diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 64b11997..6380e8f6 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -8,7 +8,7 @@ from itertools import chain, count import os from first import first -from pip9.req import InstallRequirement +from ._compat import InstallRequirement from . import click from .cache import DependencyCache @@ -302,7 +302,7 @@ class Resolver(object): dependency_strings = self.dependency_cache[ireq] log.debug(' {:25} requires {}'.format(format_requirement(ireq), ', '.join(sorted(dependency_strings, key=lambda s: s.lower())) or '-')) - from notpip._vendor.packaging.markers import InvalidMarker + from pip9._vendor.packaging.markers import InvalidMarker for dependency_string in dependency_strings: try: _dependency_string = dependency_string @@ -315,7 +315,6 @@ class Resolver(object): except InvalidMarker: yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint) - def reverse_dependencies(self, ireqs): non_editable = [ireq for ireq in ireqs if not ireq.editable] return self.dependency_cache.reverse_dependencies(non_editable) diff --git a/pipenv/patched/piptools/scripts/compile.py b/pipenv/patched/piptools/scripts/compile.py index 8194fcf9..484b710c 100644 --- a/pipenv/patched/piptools/scripts/compile.py +++ b/pipenv/patched/piptools/scripts/compile.py @@ -7,8 +7,12 @@ import os import sys import tempfile -import pip9 -from pip9.req import InstallRequirement, parse_requirements +from .._compat import ( + InstallRequirement, + parse_requirements, + cmdoptions, + Command, +) from .. import click from ..exceptions import PipToolsError @@ -21,7 +25,7 @@ from ..writer import OutputWriter DEFAULT_REQUIREMENTS_FILE = 'requirements.in' -class PipCommand(pip9.basecommand.Command): +class PipCommand(Command): name = 'PipCommand' @@ -247,12 +251,14 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, def get_pip_command(): - # Use pip's parser for pip9.conf management and defaults. + # Use pip's parser for pip.conf management and defaults. # General options (find_links, index_url, extra_index_url, trusted_host, - # and pre) are defered to pip9. + # and pre) are defered to pip. pip_command = PipCommand() - index_opts = pip9.cmdoptions.make_option_group( - pip9.cmdoptions.index_group, + pip_command.parser.add_option(cmdoptions.no_binary()) + pip_command.parser.add_option(cmdoptions.only_binary()) + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, pip_command.parser, ) pip_command.parser.insert_option_group(0, index_opts) diff --git a/pipenv/patched/piptools/scripts/sync.py b/pipenv/patched/piptools/scripts/sync.py index c43c5af8..610c1d5e 100644 --- a/pipenv/patched/piptools/scripts/sync.py +++ b/pipenv/patched/piptools/scripts/sync.py @@ -5,9 +5,9 @@ from __future__ import (absolute_import, division, print_function, import os import sys -import pip9 from .. import click, sync +from .._compat import parse_requirements, get_installed_distributions from ..exceptions import PipToolsError from ..logging import log from ..utils import flat_map @@ -45,7 +45,7 @@ def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet, log.error('ERROR: ' + msg) sys.exit(2) - requirements = flat_map(lambda src: pip9.req.parse_requirements(src, session=True), + requirements = flat_map(lambda src: parse_requirements(src, session=True), src_files) try: @@ -54,7 +54,7 @@ def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet, log.error(str(e)) sys.exit(2) - installed_dists = pip9.get_installed_distributions(skip=[], user_only=user_only) + installed_dists = get_installed_distributions(skip=[], user_only=user_only) to_install, to_uninstall = sync.diff(requirements, installed_dists) install_flags = [] diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index db8bb9b3..d76695d4 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -2,11 +2,13 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import os import sys from itertools import chain, groupby from collections import OrderedDict +from contextlib import contextmanager -from pip9.req import InstallRequirement +from ._compat import InstallRequirement from first import first @@ -240,3 +242,17 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() + + +# Borrowed from pew to avoid importing pew which imports psutil +# See https://github.com/berdario/pew/blob/master/pew/_utils.py#L82 +@contextmanager +def temp_environ(): + """Allow the ability to set os.environ temporarily""" + environ = dict(os.environ) + try: + yield + + finally: + os.environ.clear() + os.environ.update(environ) diff --git a/pipenv/patched/safety/cli.py b/pipenv/patched/safety/cli.py index fc543ef5..37ae3acb 100644 --- a/pipenv/patched/safety/cli.py +++ b/pipenv/patched/safety/cli.py @@ -15,7 +15,7 @@ try: from pip import get_installed_distributions except ImportError: # pip 10 - from pip9._internal.utils.misc import get_installed_distributions + from pip._internal.utils.misc import get_installed_distributions @click.group()