kennethreitz/pipenv
1
0
Fork 0
mirror of https://github.com/kennethreitz/pipenv.git synced 2026-06-05 22:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
avoid-cmdify
pipenv/tasks/vendoring/patches/patched/piptools.patch
T
Dan Ryan d069dff844 Update project and spinner outputs 
Signed-off-by: Dan Ryan <dan@danryan.co>

Try again

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix test config to skip failed removals

Signed-off-by: Dan Ryan <dan@danryan.co>

Update piptools to handle some errors

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix test config to skip failed removals

Signed-off-by: Dan Ryan <dan@danryan.co>

Update tempfile.py

Use vistirs temporary directory implementation

Update temp_dir.py

Force pip to use weakrefs in tempdirs

Fix pip implementation to set name of tempdir

typo fix

Signed-off-by: Dan Ryan <dan@danryan.co>

fix pip tempdir implementation

Signed-off-by: Dan Ryan <dan@danryan.co>

Update tempfiles to use weakrefs

Signed-off-by: Dan Ryan <dan@danryan.co>

fix patch paths

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix pip tempdir implementation

Signed-off-by: Dan Ryan <dan@danryan.co>

Syntax error fix

Signed-off-by: Dan Ryan <dan@danryan.co>

Unconstrain windows tests

Signed-off-by: Dan Ryan <dan@danryan.co>

Update dependencies, add news

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix pythonfinder path search nesting bug

- Fixes #3121

Signed-off-by: Dan Ryan <dan@danryan.co>

Update requirementslib

- Fix subdirectory issue

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix logic error

Signed-off-by: Dan Ryan <dan@danryan.co>

conditional builds

Signed-off-by: Dan Ryan <dan@danryan.co>
2018-10-30 01:07:26 -04:00

775 lines
32 KiB
Diff
Raw Permalink Blame History

diff --git a/pipenv/patched/piptools/_compat/__init__.py b/pipenv/patched/piptools/_compat/__init__.py
index 1fa3805..c0ecec8 100644
--- a/pipenv/patched/piptools/_compat/__init__.py
+++ b/pipenv/patched/piptools/_compat/__init__.py
@@ -27,6 +27,8 @@ from .pip_compat import (
cmdoptions,
get_installed_distributions,
PyPI,
+ SafeFileCache,
+ InstallationError,
install_req_from_line,
install_req_from_editable,
)
diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py
index 28da51f..c466ef0 100644
--- a/pipenv/patched/piptools/_compat/pip_compat.py
+++ b/pipenv/patched/piptools/_compat/pip_compat.py
@@ -1,45 +1,55 @@
# -*- coding=utf-8 -*-
-import importlib
-import pip
-import pkg_resources
+__all__ = [
+ "InstallRequirement",
+ "parse_requirements",
+ "RequirementSet",
+ "user_cache_dir",
+ "FAVORITE_HASH",
+ "is_file_url",
+ "url_to_path",
+ "PackageFinder",
+ "FormatControl",
+ "Wheel",
+ "Command",
+ "cmdoptions",
+ "get_installed_distributions",
+ "PyPI",
+ "SafeFileCache",
+ "InstallationError",
+ "parse_version",
+ "pip_version",
+ "install_req_from_editable",
+ "install_req_from_line",
+ "user_cache_dir"
+]
-def do_import(module_path, subimport=None, old_path=None):
- old_path = old_path or module_path
- prefixes = ["pip._internal", "pip"]
- paths = [module_path, old_path]
- search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None]
- package = subimport if subimport else None
- for to_import in search_order:
- if not subimport:
- to_import, _, package = to_import.rpartition(".")
- try:
- imported = importlib.import_module(to_import)
- except ImportError:
- continue
- else:
- return getattr(imported, package)
-
-
-InstallRequirement = do_import('req.req_install', 'InstallRequirement')
-parse_requirements = do_import('req.req_file', 'parse_requirements')
-RequirementSet = do_import('req.req_set', 'RequirementSet')
-user_cache_dir = do_import('utils.appdirs', 'user_cache_dir')
-FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH')
-is_file_url = do_import('download', 'is_file_url')
-url_to_path = do_import('download', 'url_to_path')
-PackageFinder = do_import('index', 'PackageFinder')
-FormatControl = do_import('index', 'FormatControl')
-Wheel = do_import('wheel', 'Wheel')
-Command = do_import('cli.base_command', 'Command', old_path='basecommand')
-cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions')
-get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils')
-PyPI = do_import('models.index', 'PyPI')
+from pipenv.vendor.appdirs import user_cache_dir
+from pip_shims.shims import (
+ InstallRequirement,
+ parse_requirements,
+ RequirementSet,
+ FAVORITE_HASH,
+ is_file_url,
+ url_to_path,
+ PackageFinder,
+ FormatControl,
+ Wheel,
+ Command,
+ cmdoptions,
+ get_installed_distributions,
+ PyPI,
+ SafeFileCache,
+ InstallationError,
+ parse_version,
+ pip_version,
+)
# pip 18.1 has refactored InstallRequirement constructors use by pip-tools.
-if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('18.1'):
+if parse_version(pip_version) < parse_version('18.1'):
install_req_from_line = InstallRequirement.from_line
install_req_from_editable = InstallRequirement.from_editable
else:
- install_req_from_line = do_import('req.constructors', 'install_req_from_line')
- install_req_from_editable = do_import('req.constructors', 'install_req_from_editable')
+ from pip_shims.shims import (
+ install_req_from_editable, install_req_from_line
+ )
diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py
index 08dabe1..480ad1e 100644
--- a/pipenv/patched/piptools/repositories/local.py
+++ b/pipenv/patched/piptools/repositories/local.py
@@ -56,7 +56,7 @@ class LocalRequirementsRepository(BaseRepository):
if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin):
project, version, _ = as_tuple(existing_pin)
return make_install_requirement(
- project, version, ireq.extras, constraint=ireq.constraint
+ project, version, ireq.extras, constraint=ireq.constraint, markers=ireq.markers
)
else:
return self.repository.find_best_match(ireq, prereleases)
diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
index bf69803..31b85b9 100644
--- a/pipenv/patched/piptools/repositories/pypi.py
+++ b/pipenv/patched/piptools/repositories/pypi.py
@@ -1,7 +1,7 @@
# coding: utf-8
from __future__ import (absolute_import, division, print_function,
unicode_literals)
-
+import copy
import hashlib
import os
from contextlib import contextmanager
@@ -15,13 +15,22 @@ from .._compat import (
Wheel,
FAVORITE_HASH,
TemporaryDirectory,
- PyPI
+ PyPI,
+ InstallRequirement,
+ SafeFileCache
)
+os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip")
+from pip_shims.shims import do_import, VcsSupport, WheelCache
+from packaging.requirements import Requirement
+from packaging.specifiers import SpecifierSet, Specifier
+InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999"))
+from pip._internal.resolve import Resolver as PipResolver
+
-from ..cache import CACHE_DIR
+from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR
from ..exceptions import NoCandidateFound
-from ..utils import (fs_str, is_pinned_requirement, lookup_table,
- make_install_requirement)
+from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup,
+ make_install_requirement, clean_requires_python)
from .base import BaseRepository
try:
@@ -31,10 +40,44 @@ except ImportError:
def RequirementTracker():
yield
-try:
- from pip._internal.cache import WheelCache
-except ImportError:
- from pip.wheel import WheelCache
+
+class HashCache(SafeFileCache):
+ """Caches hashes of PyPI artifacts so we do not need to re-download them
+
+ Hashes are only cached when the URL appears to contain a hash in it and the cache key includes
+ the hash value returned from the server). This ought to avoid ssues where the location on the
+ server changes."""
+ def __init__(self, *args, **kwargs):
+ session = kwargs.pop('session')
+ self.session = session
+ kwargs.setdefault('directory', os.path.join(CACHE_DIR, 'hash-cache'))
+ super(HashCache, self).__init__(*args, **kwargs)
+
+ def get_hash(self, location):
+ # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it
+ hash_value = None
+ vcs = VcsSupport()
+ orig_scheme = location.scheme
+ new_location = copy.deepcopy(location)
+ if orig_scheme in vcs.all_schemes:
+ new_location.url = new_location.url.split("+", 1)[-1]
+ can_hash = new_location.hash
+ if can_hash:
+ # hash url WITH fragment
+ hash_value = self.get(new_location.url)
+ if not hash_value:
+ hash_value = self._get_file_hash(new_location) if not new_location.url.startswith("ssh") else None
+ hash_value = hash_value.encode('utf8') if hash_value else None
+ if can_hash:
+ self.set(new_location.url, hash_value)
+ return hash_value.decode('utf8') if hash_value else None
+
+ def _get_file_hash(self, location):
+ h = hashlib.new(FAVORITE_HASH)
+ with open_local_or_remote_file(location, self.session) as fp:
+ for chunk in iter(lambda: fp.read(8096), b""):
+ h.update(chunk)
+ return ":".join([FAVORITE_HASH, h.hexdigest()])
class PyPIRepository(BaseRepository):
@@ -46,8 +89,9 @@ class PyPIRepository(BaseRepository):
config), but any other PyPI mirror can be used if index_urls is
changed/configured on the Finder.
"""
- def __init__(self, pip_options, session):
+ def __init__(self, pip_options, session, use_json=False):
self.session = session
+ self.use_json = use_json
self.pip_options = pip_options
index_urls = [pip_options.index_url] + pip_options.extra_index_urls
@@ -73,6 +117,10 @@ class PyPIRepository(BaseRepository):
# of all secondary dependencies for the given requirement, so we
# only have to go to disk once for each requirement
self._dependencies_cache = {}
+ self._json_dep_cache = {}
+
+ # stores *full* path + fragment => sha256
+ self._hash_cache = HashCache(session=session)
# Setup file paths
self.freshen_build_caches()
@@ -113,10 +161,13 @@ class PyPIRepository(BaseRepository):
if ireq.editable:
return ireq # return itself as the best match
- all_candidates = self.find_all_candidates(ireq.name)
+ all_candidates = clean_requires_python(self.find_all_candidates(ireq.name))
candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
- matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
+ try:
+ matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
prereleases=prereleases)
+ except TypeError:
+ matching_versions = [candidate.version for candidate in all_candidates]
# Reuses pip's internal candidate sort key to sort
matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
@@ -126,25 +177,87 @@ class PyPIRepository(BaseRepository):
# Turn the candidate into a pinned InstallRequirement
return make_install_requirement(
- best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint
- )
+ best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint
+ )
+
+ def get_json_dependencies(self, ireq):
+
+ if not (is_pinned_requirement(ireq)):
+ raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq))
+
+ def gen(ireq):
+ if self.DEFAULT_INDEX_URL not in self.finder.index_urls:
+ return
+
+ url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name)
+ releases = self.session.get(url).json()['releases']
+
+ matches = [
+ r for r in releases
+ if '=={0}'.format(r) == str(ireq.req.specifier)
+ ]
+ if not matches:
+ return
+
+ release_requires = self.session.get(
+ 'https://pypi.org/pypi/{0}/{1}/json'.format(
+ ireq.req.name, matches[0],
+ ),
+ ).json()
+ try:
+ requires_dist = release_requires['info']['requires_dist']
+ except KeyError:
+ return
+
+ for requires in requires_dist:
+ i = InstallRequirement.from_line(requires)
+ if 'extra' not in repr(i.markers):
+ yield i
+
+ try:
+ if ireq not in self._json_dep_cache:
+ self._json_dep_cache[ireq] = [g for g in gen(ireq)]
- def resolve_reqs(self, download_dir, ireq, wheel_cache):
+ return set(self._json_dep_cache[ireq])
+ except Exception:
+ return set()
+
+ def get_dependencies(self, ireq):
+ json_results = set()
+
+ if self.use_json:
+ try:
+ json_results = self.get_json_dependencies(ireq)
+ except TypeError:
+ json_results = set()
+
+ legacy_results = self.get_legacy_dependencies(ireq)
+ json_results.update(legacy_results)
+
+ return json_results
+
+ def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist=None):
results = None
+ setup_requires = {}
+ dist = None
+ ireq.isolated = False
+ ireq._wheel_cache = wheel_cache
+
try:
from pip._internal.operations.prepare import RequirementPreparer
- from pip._internal.resolve import Resolver as PipResolver
except ImportError:
- # Pip 9 and below
+ # Pip 9 and below
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
+ ignore_installed=True,
+ ignore_compatibility=False,
wheel_cache=wheel_cache
)
- results = reqset._prepare_file(self.finder, ireq)
+ results = reqset._prepare_file(self.finder, ireq, ignore_requires_python=True)
else:
# pip >= 10
preparer_kwargs = {
@@ -159,13 +272,14 @@ class PyPIRepository(BaseRepository):
'finder': self.finder,
'session': self.session,
'upgrade_strategy': "to-satisfy-only",
- 'force_reinstall': False,
+ 'force_reinstall': True,
'ignore_dependencies': False,
- 'ignore_requires_python': False,
+ 'ignore_requires_python': True,
'ignore_installed': True,
'isolated': False,
'wheel_cache': wheel_cache,
- 'use_user_site': False
+ 'use_user_site': False,
+ 'ignore_compatibility': False
}
resolver = None
preparer = None
@@ -177,15 +291,109 @@ class PyPIRepository(BaseRepository):
resolver_kwargs['preparer'] = preparer
reqset = RequirementSet()
ireq.is_direct = True
- reqset.add_requirement(ireq)
+ # reqset.add_requirement(ireq)
resolver = PipResolver(**resolver_kwargs)
resolver.require_hashes = False
results = resolver._resolve_one(reqset, ireq)
- reqset.cleanup_files()
- return set(results)
+ cleanup_fn = getattr(reqset, "cleanup_files", None)
+ if cleanup_fn is not None:
+ try:
+ cleanup_fn()
+ except OSError:
+ pass
+
+ if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)):
+ if ireq.editable:
+ self._source_dir = TemporaryDirectory(fs_str("source"))
+ ireq.ensure_has_source_dir(self.source_dir)
+
+ if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)):
+ # Collect setup_requires info from local eggs.
+ # Do this after we call the preparer on these reqs to make sure their
+ # egg info has been created
+ from pipenv.utils import chdir
+ with chdir(ireq.setup_py_dir):
+ try:
+ from setuptools.dist import distutils
+ dist = distutils.core.run_setup(ireq.setup_py)
+ except InstallationError:
+ ireq.run_egg_info()
+ except (TypeError, ValueError, AttributeError):
+ pass
+ if not dist:
+ try:
+ dist = ireq.get_dist()
+ except (ImportError, ValueError, TypeError, AttributeError):
+ pass
+ if ireq.editable and dist:
+ setup_requires = getattr(dist, "extras_require", None)
+ if not setup_requires:
+ setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)}
+ if not getattr(ireq, 'req', None):
+ try:
+ ireq.req = dist.as_requirement() if dist else None
+ except (ValueError, TypeError) as e:
+ pass
- def get_dependencies(self, ireq):
+ # Convert setup_requires dict into a somewhat usable form.
+ if setup_requires:
+ for section in setup_requires:
+ python_version = section
+ not_python = not (section.startswith('[') and ':' in section)
+
+ # This is for cleaning up :extras: formatted markers
+ # by adding them to the results of the resolver
+ # since any such extra would have been returned as a result anyway
+ for value in setup_requires[section]:
+ # This is a marker.
+ if value.startswith('[') and ':' in value:
+ python_version = value[1:-1]
+ not_python = False
+ # Strip out other extras.
+ if value.startswith('[') and ':' not in value:
+ not_python = True
+
+ if ':' not in value:
+ try:
+ if not not_python:
+ results.add(InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';')))
+ # Anything could go wrong here -- can't be too careful.
+ except Exception:
+ pass
+
+ # this section properly creates 'python_version' markers for cross-python
+ # virtualenv creation and for multi-python compatibility.
+ requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else resolver.requires_python
+ if requires_python:
+ marker_str = ''
+ # This corrects a logic error from the previous code which said that if
+ # we Encountered any 'requires_python' attributes, basically only create a
+ # single result no matter how many we resolved. This should fix
+ # a majority of the remaining non-deterministic resolution issues.
+ if any(requires_python.startswith(op) for op in Specifier._operators.keys()):
+ # We are checking first if we have leading specifier operator
+ # if not, we can assume we should be doing a == comparison
+ specifierset = SpecifierSet(requires_python)
+ # for multiple specifiers, the correct way to represent that in
+ # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
+ from passa.internals.specifiers import cleanup_pyspecs
+ marker_str = str(Marker(" and ".join(dedup([
+ "python_version {0[0]} '{0[1]}'".format(spec)
+ for spec in cleanup_pyspecs(specifierset)
+ ]))))
+ # The best way to add markers to a requirement is to make a separate requirement
+ # with only markers on it, and then to transfer the object istelf
+ marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker
+ if ireq in results:
+ results.remove(ireq)
+ print(marker_to_add)
+ ireq.req.marker = marker_to_add
+
+ results = set(results) if results else set()
+ return results, ireq
+
+ def get_legacy_dependencies(self, ireq):
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
@@ -200,6 +408,7 @@ class PyPIRepository(BaseRepository):
# If a download_dir is passed, pip will unnecessarely
# archive the entire source directory
download_dir = None
+
elif ireq.link and not ireq.link.is_artifact:
# No download_dir for VCS sources. This also works around pip
# using git-checkout-index, which gets rid of the .git dir.
@@ -214,7 +423,8 @@ class PyPIRepository(BaseRepository):
wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control)
prev_tracker = os.environ.get('PIP_REQ_TRACKER')
try:
- self._dependencies_cache[ireq] = self.resolve_reqs(download_dir, ireq, wheel_cache)
+ results, ireq = self.resolve_reqs(download_dir, ireq, wheel_cache)
+ self._dependencies_cache[ireq] = results
finally:
if 'PIP_REQ_TRACKER' in os.environ:
if prev_tracker:
@@ -236,6 +446,10 @@ class PyPIRepository(BaseRepository):
if ireq.editable:
return set()
+ vcs = VcsSupport()
+ if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme:
+ return set()
+
if not is_pinned_requirement(ireq):
raise TypeError(
"Expected pinned requirement, got {}".format(ireq))
@@ -243,24 +457,22 @@ class PyPIRepository(BaseRepository):
# We need to get all of the candidates that match our current version
# pin, these will represent all of the files that could possibly
# satisfy this constraint.
- all_candidates = self.find_all_candidates(ireq.name)
- candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
- matching_versions = list(
- ireq.specifier.filter((candidate.version for candidate in all_candidates)))
- matching_candidates = candidates_by_version[matching_versions[0]]
+ ### Modification -- this is much more efficient....
+ ### modification again -- still more efficient
+ matching_candidates = (
+ c for c in clean_requires_python(self.find_all_candidates(ireq.name))
+ if c.version in ireq.specifier
+ )
+ # candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
+ # matching_versions = list(
+ # ireq.specifier.filter((candidate.version for candidate in all_candidates)))
+ # matching_candidates = candidates_by_version[matching_versions[0]]
return {
- self._get_file_hash(candidate.location)
- for candidate in matching_candidates
+ h for h in map(lambda c: self._hash_cache.get_hash(c.location),
+ matching_candidates) if h is not None
}
- def _get_file_hash(self, location):
- h = hashlib.new(FAVORITE_HASH)
- with open_local_or_remote_file(location, self.session) as fp:
- for chunk in iter(lambda: fp.read(8096), b""):
- h.update(chunk)
- return ":".join([FAVORITE_HASH, h.hexdigest()])
-
@contextmanager
def allow_all_wheels(self):
"""
diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py
index c2d323c..d5a471d 100644
--- a/pipenv/patched/piptools/resolver.py
+++ b/pipenv/patched/piptools/resolver.py
@@ -13,7 +13,7 @@ from . import click
from .cache import DependencyCache
from .exceptions import UnsupportedConstraint
from .logging import log
-from .utils import (format_requirement, format_specifier, full_groupby,
+from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers,
is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES)
green = partial(click.style, fg='green')
@@ -27,6 +27,7 @@ class RequirementSummary(object):
def __init__(self, ireq):
self.req = ireq.req
self.key = key_from_req(ireq.req)
+ self.markers = ireq.markers
self.extras = str(sorted(ireq.extras))
self.specifier = str(ireq.specifier)
@@ -119,7 +120,7 @@ class Resolver(object):
@staticmethod
def check_constraints(constraints):
for constraint in constraints:
- if constraint.link is not None and not constraint.editable:
+ if constraint.link is not None and not constraint.editable and not constraint.is_wheel:
msg = ('pip-compile does not support URLs as packages, unless they are editable. '
'Perhaps add -e option?')
raise UnsupportedConstraint(msg, constraint)
@@ -155,6 +156,13 @@ class Resolver(object):
# NOTE we may be losing some info on dropped reqs here
combined_ireq.req.specifier &= ireq.req.specifier
combined_ireq.constraint &= ireq.constraint
+ if not combined_ireq.markers:
+ combined_ireq.markers = ireq.markers
+ else:
+ _markers = combined_ireq.markers._markers
+ if not isinstance(_markers[0], (tuple, list)):
+ combined_ireq.markers._markers = [_markers, 'and', ireq.markers._markers]
+
# Return a sorted, de-duped tuple of extras
combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras))))
yield combined_ireq
@@ -272,6 +280,15 @@ class Resolver(object):
for dependency in self.repository.get_dependencies(ireq):
yield dependency
return
+
+ # fix our malformed extras
+ if ireq.extras:
+ if getattr(ireq, "extra", None):
+ if ireq.extras:
+ ireq.extras.extend(ireq.extra)
+ else:
+ ireq.extras = ireq.extra
+
elif not is_pinned_requirement(ireq):
raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq))
@@ -282,7 +299,7 @@ class Resolver(object):
if ireq not in self.dependency_cache:
log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow')
dependencies = self.repository.get_dependencies(ireq)
- self.dependency_cache[ireq] = sorted(str(ireq.req) for ireq in dependencies)
+ self.dependency_cache[ireq] = sorted(set(format_requirement(ireq) for ireq in dependencies))
# Example: ['Werkzeug>=0.9', 'Jinja2>=2.4']
dependency_strings = self.dependency_cache[ireq]
diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py
index 2360a04..6f62eb9 100644
--- a/pipenv/patched/piptools/utils.py
+++ b/pipenv/patched/piptools/utils.py
@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function,
import os
import sys
+import six
from itertools import chain, groupby
from collections import OrderedDict
from contextlib import contextmanager
@@ -11,11 +12,78 @@ from contextlib import contextmanager
from ._compat import install_req_from_line
from .click import style
+from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier
+from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
+from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable
UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'}
+def simplify_markers(ireq):
+ """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`"
+
+ Clean and deduplicate markers.
+
+ :param ireq: An InstallRequirement to clean
+ :type ireq: :class:`~pip._internal.req.req_install.InstallRequirement`
+ :return: An InstallRequirement with cleaned Markers
+ :rtype: :class:`~pip._internal.req.req_install.InstallRequirement`
+ """
+
+ if not getattr(ireq, 'markers', None):
+ return ireq
+ markers = ireq.markers
+ marker_list = []
+ if isinstance(markers, six.string_types):
+ if ';' in markers:
+ markers = [Marker(m_str.strip()) for m_str in markers.split(';')]
+ else:
+ markers = Marker(markers)
+ for m in markers._markers:
+ _single_marker = []
+ if isinstance(m[0], six.string_types):
+ continue
+ if not isinstance(m[0], (list, tuple)):
+ marker_list.append(''.join([_piece.serialize() for _piece in m]))
+ continue
+ for _marker_part in m:
+ if isinstance(_marker_part, six.string_types):
+ _single_marker.append(_marker_part)
+ continue
+ _single_marker.append(''.join([_piece.serialize() for _piece in _marker_part]))
+ _single_marker = [_m.strip() for _m in _single_marker]
+ marker_list.append(tuple(_single_marker,))
+ marker_str = ' and '.join(list(dedup(tuple(marker_list,)))) if marker_list else ''
+ new_markers = Marker(marker_str)
+ ireq.markers = new_markers
+ new_ireq = install_req_from_line(format_requirement(ireq))
+ if ireq.constraint:
+ new_ireq.constraint = ireq.constraint
+ return new_ireq
+
+
+def clean_requires_python(candidates):
+ """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes."""
+ all_candidates = []
+ py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
+ for c in candidates:
+ if c.requires_python:
+ # Old specifications had people setting this to single digits
+ # which is effectively the same as '>=digit,<digit+1'
+ if c.requires_python.isdigit():
+ c.requires_python = '>={0},<{1}'.format(c.requires_python, int(c.requires_python) + 1)
+ try:
+ specifierset = SpecifierSet(c.requires_python)
+ except InvalidSpecifier:
+ continue
+ else:
+ if not specifierset.contains(py_version):
+ continue
+ all_candidates.append(c)
+ return all_candidates
+
+
def key_from_ireq(ireq):
"""Get a standardized key for an InstallRequirement."""
if ireq.req is None and ireq.link is not None:
@@ -41,30 +109,61 @@ def comment(text):
return style(text, fg='green')
-def make_install_requirement(name, version, extras, constraint=False):
+def make_install_requirement(name, version, extras, markers, constraint=False):
# If no extras are specified, the extras string is blank
extras_string = ""
if extras:
# Sort extras for stability
extras_string = "[{}]".format(",".join(sorted(extras)))
- return install_req_from_line(
- str('{}{}=={}'.format(name, extras_string, version)),
- constraint=constraint)
+ if not markers:
+ return install_req_from_line(
+ str('{}{}=={}'.format(name, extras_string, version)),
+ constraint=constraint)
+ else:
+ return install_req_from_line(
+ str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))),
+ constraint=constraint)
-def format_requirement(ireq, marker=None):
+def _requirement_to_str_lowercase_name(requirement):
"""
- Generic formatter for pretty printing InstallRequirements to the terminal
- in a less verbose way than using its `__str__` method.
+ Formats a packaging.requirements.Requirement with a lowercase name.
+
+ This is simply a copy of
+ https://github.com/pypa/packaging/blob/16.8/packaging/requirements.py#L109-L124
+ modified to lowercase the dependency name.
+
+ Previously, we were invoking the original Requirement.__str__ method and
+ lowercasing the entire result, which would lowercase the name, *and* other,
+ important stuff that should not be lowercased (such as the marker). See
+ this issue for more information: https://github.com/pypa/pipenv/issues/2113.
"""
+ parts = [requirement.name.lower()]
+
+ if requirement.extras:
+ parts.append("[{0}]".format(",".join(sorted(requirement.extras))))
+
+ if requirement.specifier:
+ parts.append(str(requirement.specifier))
+
+ if requirement.url:
+ parts.append("@ {0}".format(requirement.url))
+
+ if requirement.marker:
+ parts.append("; {0}".format(requirement.marker))
+
+ return "".join(parts)
+
+
+def format_requirement(ireq, marker=None):
if ireq.editable:
line = '-e {}'.format(ireq.link)
else:
- line = str(ireq.req).lower()
+ line = _requirement_to_str_lowercase_name(ireq.req)
- if marker:
- line = '{} ; {}'.format(line, marker)
+ if marker and ';' not in line:
+ line = '{}; {}'.format(line, marker)
return line
Reference in New Issue View Git Blame Copy Permalink