diff --git a/pipenv/patched/piptools/_compat/__init__.py b/pipenv/patched/piptools/_compat/__init__.py index 1fa3805..c0ecec8 100644 --- a/pipenv/patched/piptools/_compat/__init__.py +++ b/pipenv/patched/piptools/_compat/__init__.py @@ -27,6 +27,8 @@ from .pip_compat import ( cmdoptions, get_installed_distributions, PyPI, + SafeFileCache, + InstallationError, install_req_from_line, install_req_from_editable, ) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py index 28da51f..c466ef0 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -1,45 +1,55 @@ # -*- coding=utf-8 -*- -import importlib -import pip -import pkg_resources +__all__ = [ + "InstallRequirement", + "parse_requirements", + "RequirementSet", + "user_cache_dir", + "FAVORITE_HASH", + "is_file_url", + "url_to_path", + "PackageFinder", + "FormatControl", + "Wheel", + "Command", + "cmdoptions", + "get_installed_distributions", + "PyPI", + "SafeFileCache", + "InstallationError", + "parse_version", + "pip_version", + "install_req_from_editable", + "install_req_from_line", + "user_cache_dir" +] -def do_import(module_path, subimport=None, old_path=None): - old_path = old_path or module_path - prefixes = ["pip._internal", "pip"] - paths = [module_path, old_path] - search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] - package = subimport if subimport else None - for to_import in search_order: - if not subimport: - to_import, _, package = to_import.rpartition(".") - try: - imported = importlib.import_module(to_import) - except ImportError: - continue - else: - return getattr(imported, package) - - -InstallRequirement = do_import('req.req_install', 'InstallRequirement') -parse_requirements = do_import('req.req_file', 'parse_requirements') -RequirementSet = do_import('req.req_set', 'RequirementSet') -user_cache_dir = do_import('utils.appdirs', 'user_cache_dir') -FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH') -is_file_url = do_import('download', 'is_file_url') -url_to_path = do_import('download', 'url_to_path') -PackageFinder = do_import('index', 'PackageFinder') -FormatControl = do_import('index', 'FormatControl') -Wheel = do_import('wheel', 'Wheel') -Command = do_import('cli.base_command', 'Command', old_path='basecommand') -cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions') -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') -PyPI = do_import('models.index', 'PyPI') +from pipenv.vendor.appdirs import user_cache_dir +from pip_shims.shims import ( + InstallRequirement, + parse_requirements, + RequirementSet, + FAVORITE_HASH, + is_file_url, + url_to_path, + PackageFinder, + FormatControl, + Wheel, + Command, + cmdoptions, + get_installed_distributions, + PyPI, + SafeFileCache, + InstallationError, + parse_version, + pip_version, +) # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('18.1'): +if parse_version(pip_version) < parse_version('18.1'): install_req_from_line = InstallRequirement.from_line install_req_from_editable = InstallRequirement.from_editable else: - install_req_from_line = do_import('req.constructors', 'install_req_from_line') - install_req_from_editable = do_import('req.constructors', 'install_req_from_editable') + from pip_shims.shims import ( + install_req_from_editable, install_req_from_line + ) diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py index 08dabe1..480ad1e 100644 --- a/pipenv/patched/piptools/repositories/local.py +++ b/pipenv/patched/piptools/repositories/local.py @@ -56,7 +56,7 @@ class LocalRequirementsRepository(BaseRepository): if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin): project, version, _ = as_tuple(existing_pin) return make_install_requirement( - project, version, ireq.extras, constraint=ireq.constraint + project, version, ireq.extras, constraint=ireq.constraint, markers=ireq.markers ) else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index bf69803..31b85b9 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,7 +1,7 @@ # coding: utf-8 from __future__ import (absolute_import, division, print_function, unicode_literals) - +import copy import hashlib import os from contextlib import contextmanager @@ -15,13 +15,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, - PyPI + PyPI, + InstallRequirement, + SafeFileCache ) +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") +from pip_shims.shims import do_import, VcsSupport, WheelCache +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet, Specifier +InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) +from pip._internal.resolve import Resolver as PipResolver + -from ..cache import CACHE_DIR +from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR from ..exceptions import NoCandidateFound -from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, + make_install_requirement, clean_requires_python) from .base import BaseRepository try: @@ -31,10 +40,44 @@ except ImportError: def RequirementTracker(): yield -try: - from pip._internal.cache import WheelCache -except ImportError: - from pip.wheel import WheelCache + +class HashCache(SafeFileCache): + """Caches hashes of PyPI artifacts so we do not need to re-download them + + Hashes are only cached when the URL appears to contain a hash in it and the cache key includes + the hash value returned from the server). This ought to avoid ssues where the location on the + server changes.""" + def __init__(self, *args, **kwargs): + session = kwargs.pop('session') + self.session = session + kwargs.setdefault('directory', os.path.join(CACHE_DIR, 'hash-cache')) + super(HashCache, self).__init__(*args, **kwargs) + + def get_hash(self, location): + # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it + hash_value = None + vcs = VcsSupport() + orig_scheme = location.scheme + new_location = copy.deepcopy(location) + if orig_scheme in vcs.all_schemes: + new_location.url = new_location.url.split("+", 1)[-1] + can_hash = new_location.hash + if can_hash: + # hash url WITH fragment + hash_value = self.get(new_location.url) + if not hash_value: + hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None + hash_value = hash_value.encode('utf8') if hash_value else None + if can_hash: + self.set(new_location.url, hash_value) + return hash_value.decode('utf8') if hash_value else None + + def _get_file_hash(self, location): + h = hashlib.new(FAVORITE_HASH) + with open_local_or_remote_file(location, self.session) as fp: + for chunk in iter(lambda: fp.read(8096), b""): + h.update(chunk) + return ":".join([FAVORITE_HASH, h.hexdigest()]) class PyPIRepository(BaseRepository): @@ -46,8 +89,9 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ - def __init__(self, pip_options, session): + def __init__(self, pip_options, session, use_json=False): self.session = session + self.use_json = use_json self.pip_options = pip_options index_urls = [pip_options.index_url] + pip_options.extra_index_urls @@ -73,6 +117,10 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} + self._json_dep_cache = {} + + # stores *full* path + fragment => sha256 + self._hash_cache = HashCache(session=session) # Setup file paths self.freshen_build_caches() @@ -113,10 +161,13 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match - all_candidates = self.find_all_candidates(ireq.name) + all_candidates = clean_requires_python(self.find_all_candidates(ireq.name)) candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) - matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), + try: + matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), prereleases=prereleases) + except TypeError: + matching_versions = [candidate.version for candidate in all_candidates] # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] @@ -126,25 +177,87 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( - best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint - ) + best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint + ) + + def get_json_dependencies(self, ireq): + + if not (is_pinned_requirement(ireq)): + raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq)) + + def gen(ireq): + if self.DEFAULT_INDEX_URL not in self.finder.index_urls: + return + + url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) + releases = self.session.get(url).json()['releases'] + + matches = [ + r for r in releases + if '=={0}'.format(r) == str(ireq.req.specifier) + ] + if not matches: + return + + release_requires = self.session.get( + 'https://pypi.org/pypi/{0}/{1}/json'.format( + ireq.req.name, matches[0], + ), + ).json() + try: + requires_dist = release_requires['info']['requires_dist'] + except KeyError: + return + + for requires in requires_dist: + i = InstallRequirement.from_line(requires) + if 'extra' not in repr(i.markers): + yield i + + try: + if ireq not in self._json_dep_cache: + self._json_dep_cache[ireq] = [g for g in gen(ireq)] - def resolve_reqs(self, download_dir, ireq, wheel_cache): + return set(self._json_dep_cache[ireq]) + except Exception: + return set() + + def get_dependencies(self, ireq): + json_results = set() + + if self.use_json: + try: + json_results = self.get_json_dependencies(ireq) + except TypeError: + json_results = set() + + legacy_results = self.get_legacy_dependencies(ireq) + json_results.update(legacy_results) + + return json_results + + def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist=None): results = None + setup_requires = {} + dist = None + ireq.isolated = False + ireq._wheel_cache = wheel_cache + try: from pip._internal.operations.prepare import RequirementPreparer - from pip._internal.resolve import Resolver as PipResolver except ImportError: - # Pip 9 and below + # Pip 9 and below reqset = RequirementSet( self.build_dir, self.source_dir, download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, + ignore_installed=True, + ignore_compatibility=False, wheel_cache=wheel_cache ) - results = reqset._prepare_file(self.finder, ireq) + results = reqset._prepare_file(self.finder, ireq, ignore_requires_python=True) else: # pip >= 10 preparer_kwargs = { @@ -159,13 +272,14 @@ class PyPIRepository(BaseRepository): 'finder': self.finder, 'session': self.session, 'upgrade_strategy': "to-satisfy-only", - 'force_reinstall': False, + 'force_reinstall': True, 'ignore_dependencies': False, - 'ignore_requires_python': False, + 'ignore_requires_python': True, 'ignore_installed': True, 'isolated': False, 'wheel_cache': wheel_cache, - 'use_user_site': False + 'use_user_site': False, + 'ignore_compatibility': False } resolver = None preparer = None @@ -177,15 +291,109 @@ class PyPIRepository(BaseRepository): resolver_kwargs['preparer'] = preparer reqset = RequirementSet() ireq.is_direct = True - reqset.add_requirement(ireq) + # reqset.add_requirement(ireq) resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() - return set(results) + cleanup_fn = getattr(reqset, "cleanup_files", None) + if cleanup_fn is not None: + try: + cleanup_fn() + except OSError: + pass + + if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): + if ireq.editable: + self._source_dir = TemporaryDirectory(fs_str("source")) + ireq.ensure_has_source_dir(self.source_dir) + + if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): + # Collect setup_requires info from local eggs. + # Do this after we call the preparer on these reqs to make sure their + # egg info has been created + from pipenv.utils import chdir + with chdir(ireq.setup_py_dir): + try: + from setuptools.dist import distutils + dist = distutils.core.run_setup(ireq.setup_py) + except InstallationError: + ireq.run_egg_info() + except (TypeError, ValueError, AttributeError): + pass + if not dist: + try: + dist = ireq.get_dist() + except (ImportError, ValueError, TypeError, AttributeError): + pass + if ireq.editable and dist: + setup_requires = getattr(dist, "extras_require", None) + if not setup_requires: + setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)} + if not getattr(ireq, 'req', None): + try: + ireq.req = dist.as_requirement() if dist else None + except (ValueError, TypeError) as e: + pass - def get_dependencies(self, ireq): + # Convert setup_requires dict into a somewhat usable form. + if setup_requires: + for section in setup_requires: + python_version = section + not_python = not (section.startswith('[') and ':' in section) + + # This is for cleaning up :extras: formatted markers + # by adding them to the results of the resolver + # since any such extra would have been returned as a result anyway + for value in setup_requires[section]: + # This is a marker. + if value.startswith('[') and ':' in value: + python_version = value[1:-1] + not_python = False + # Strip out other extras. + if value.startswith('[') and ':' not in value: + not_python = True + + if ':' not in value: + try: + if not not_python: + results.add(InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))) + # Anything could go wrong here -- can't be too careful. + except Exception: + pass + + # this section properly creates 'python_version' markers for cross-python + # virtualenv creation and for multi-python compatibility. + requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else resolver.requires_python + if requires_python: + marker_str = '' + # This corrects a logic error from the previous code which said that if + # we Encountered any 'requires_python' attributes, basically only create a + # single result no matter how many we resolved. This should fix + # a majority of the remaining non-deterministic resolution issues. + if any(requires_python.startswith(op) for op in Specifier._operators.keys()): + # We are checking first if we have leading specifier operator + # if not, we can assume we should be doing a == comparison + specifierset = SpecifierSet(requires_python) + # for multiple specifiers, the correct way to represent that in + # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')` + from passa.internals.specifiers import cleanup_pyspecs + marker_str = str(Marker(" and ".join(dedup([ + "python_version {0[0]} '{0[1]}'".format(spec) + for spec in cleanup_pyspecs(specifierset) + ])))) + # The best way to add markers to a requirement is to make a separate requirement + # with only markers on it, and then to transfer the object istelf + marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker + if ireq in results: + results.remove(ireq) + print(marker_to_add) + ireq.req.marker = marker_to_add + + results = set(results) if results else set() + return results, ireq + + def get_legacy_dependencies(self, ireq): """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). @@ -200,6 +408,7 @@ class PyPIRepository(BaseRepository): # If a download_dir is passed, pip will unnecessarely # archive the entire source directory download_dir = None + elif ireq.link and not ireq.link.is_artifact: # No download_dir for VCS sources. This also works around pip # using git-checkout-index, which gets rid of the .git dir. @@ -214,7 +423,8 @@ class PyPIRepository(BaseRepository): wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) prev_tracker = os.environ.get('PIP_REQ_TRACKER') try: - self._dependencies_cache[ireq] = self.resolve_reqs(download_dir, ireq, wheel_cache) + results, ireq = self.resolve_reqs(download_dir, ireq, wheel_cache) + self._dependencies_cache[ireq] = results finally: if 'PIP_REQ_TRACKER' in os.environ: if prev_tracker: @@ -236,6 +446,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() + vcs = VcsSupport() + if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + return set() + if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) @@ -243,24 +457,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. - all_candidates = self.find_all_candidates(ireq.name) - candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) - matching_versions = list( - ireq.specifier.filter((candidate.version for candidate in all_candidates))) - matching_candidates = candidates_by_version[matching_versions[0]] + ### Modification -- this is much more efficient.... + ### modification again -- still more efficient + matching_candidates = ( + c for c in clean_requires_python(self.find_all_candidates(ireq.name)) + if c.version in ireq.specifier + ) + # candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) + # matching_versions = list( + # ireq.specifier.filter((candidate.version for candidate in all_candidates))) + # matching_candidates = candidates_by_version[matching_versions[0]] return { - self._get_file_hash(candidate.location) - for candidate in matching_candidates + h for h in map(lambda c: self._hash_cache.get_hash(c.location), + matching_candidates) if h is not None } - def _get_file_hash(self, location): - h = hashlib.new(FAVORITE_HASH) - with open_local_or_remote_file(location, self.session) as fp: - for chunk in iter(lambda: fp.read(8096), b""): - h.update(chunk) - return ":".join([FAVORITE_HASH, h.hexdigest()]) - @contextmanager def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index c2d323c..d5a471d 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -13,7 +13,7 @@ from . import click from .cache import DependencyCache from .exceptions import UnsupportedConstraint from .logging import log -from .utils import (format_requirement, format_specifier, full_groupby, +from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) green = partial(click.style, fg='green') @@ -27,6 +27,7 @@ class RequirementSummary(object): def __init__(self, ireq): self.req = ireq.req self.key = key_from_req(ireq.req) + self.markers = ireq.markers self.extras = str(sorted(ireq.extras)) self.specifier = str(ireq.specifier) @@ -119,7 +120,7 @@ class Resolver(object): @staticmethod def check_constraints(constraints): for constraint in constraints: - if constraint.link is not None and not constraint.editable: + if constraint.link is not None and not constraint.editable and not constraint.is_wheel: msg = ('pip-compile does not support URLs as packages, unless they are editable. ' 'Perhaps add -e option?') raise UnsupportedConstraint(msg, constraint) @@ -155,6 +156,13 @@ class Resolver(object): # NOTE we may be losing some info on dropped reqs here combined_ireq.req.specifier &= ireq.req.specifier combined_ireq.constraint &= ireq.constraint + if not combined_ireq.markers: + combined_ireq.markers = ireq.markers + else: + _markers = combined_ireq.markers._markers + if not isinstance(_markers[0], (tuple, list)): + combined_ireq.markers._markers = [_markers, 'and', ireq.markers._markers] + # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq @@ -272,6 +280,15 @@ class Resolver(object): for dependency in self.repository.get_dependencies(ireq): yield dependency return + + # fix our malformed extras + if ireq.extras: + if getattr(ireq, "extra", None): + if ireq.extras: + ireq.extras.extend(ireq.extra) + else: + ireq.extras = ireq.extra + elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) @@ -282,7 +299,7 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) - self.dependency_cache[ireq] = sorted(str(ireq.req) for ireq in dependencies) + self.dependency_cache[ireq] = sorted(set(format_requirement(ireq) for ireq in dependencies)) # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4'] dependency_strings = self.dependency_cache[ireq] diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index 2360a04..6f62eb9 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, import os import sys +import six from itertools import chain, groupby from collections import OrderedDict from contextlib import contextmanager @@ -11,11 +12,78 @@ from contextlib import contextmanager from ._compat import install_req_from_line from .click import style +from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier +from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} +def simplify_markers(ireq): + """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`" + + Clean and deduplicate markers. + + :param ireq: An InstallRequirement to clean + :type ireq: :class:`~pip._internal.req.req_install.InstallRequirement` + :return: An InstallRequirement with cleaned Markers + :rtype: :class:`~pip._internal.req.req_install.InstallRequirement` + """ + + if not getattr(ireq, 'markers', None): + return ireq + markers = ireq.markers + marker_list = [] + if isinstance(markers, six.string_types): + if ';' in markers: + markers = [Marker(m_str.strip()) for m_str in markers.split(';')] + else: + markers = Marker(markers) + for m in markers._markers: + _single_marker = [] + if isinstance(m[0], six.string_types): + continue + if not isinstance(m[0], (list, tuple)): + marker_list.append(''.join([_piece.serialize() for _piece in m])) + continue + for _marker_part in m: + if isinstance(_marker_part, six.string_types): + _single_marker.append(_marker_part) + continue + _single_marker.append(''.join([_piece.serialize() for _piece in _marker_part])) + _single_marker = [_m.strip() for _m in _single_marker] + marker_list.append(tuple(_single_marker,)) + marker_str = ' and '.join(list(dedup(tuple(marker_list,)))) if marker_list else '' + new_markers = Marker(marker_str) + ireq.markers = new_markers + new_ireq = install_req_from_line(format_requirement(ireq)) + if ireq.constraint: + new_ireq.constraint = ireq.constraint + return new_ireq + + +def clean_requires_python(candidates): + """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" + all_candidates = [] + py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3])))) + for c in candidates: + if c.requires_python: + # Old specifications had people setting this to single digits + # which is effectively the same as '>=digit,