diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 8ccf025a..68d8069d 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -29,6 +29,11 @@ except ImportError: _types.add(type(arg)) return _types.pop() +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + try: from weakref import finalize @@ -51,8 +56,6 @@ if six.PY2: class ResourceWarning(Warning): pass -# -*- coding=utf-8 -*- - def pip_import(module_path, subimport=None, old_path=None): internal = 'pip._internal.{0}'.format(module_path) diff --git a/pipenv/core.py b/pipenv/core.py index 3c40da4e..b6c2790e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -46,10 +46,12 @@ from .utils import ( rmtree, split_argument, extract_uri_from_vcs_dep, + fs_str, ) from ._compat import ( TemporaryDirectory, - vcs + vcs, + Path ) from .import pep508checker, progress from .environments import ( @@ -73,6 +75,7 @@ from .environments import ( PIPENV_SHELL, PIPENV_PYTHON, PIPENV_VIRTUALENV, + PIPENV_CACHE_DIR, ) # Backport required for earlier versions of Python. @@ -1481,7 +1484,7 @@ def pip_install( pre = '--pre' if pre else '' quoted_pip = which_pip(allow_global=allow_global) quoted_pip = escape_grouped_arguments(quoted_pip) - upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else '' + upgrade_strategy = '--upgrade --upgrade-strategy=to-satisfy-only' if selective_upgrade else '' pip_command = '{0} install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format( quoted_pip, install_reqs, @@ -1494,11 +1497,23 @@ def pip_install( ) if verbose: click.echo('$ {0}'.format(pip_command), err=True) - c = delegator.run(pip_command, block=block) + cache_dir = Path(PIPENV_CACHE_DIR) + pip_config = { + 'PIP_CACHE_DIR': fs_str(cache_dir.as_posix()), + 'PIP_WHEEL_DIR': fs_str(cache_dir.joinpath('wheels').as_posix()), + 'PIP_DESTINATION_DIR': fs_str(cache_dir.joinpath('pkgs').as_posix()), + } + c = delegator.run(pip_command, block=block, env=pip_config) return c def pip_download(package_name): + cache_dir = Path(PIPENV_CACHE_DIR) + pip_config = { + 'PIP_CACHE_DIR': fs_str(cache_dir.as_posix()), + 'PIP_WHEEL_DIR': fs_str(cache_dir.joinpath('wheels').as_posix()), + 'PIP_DESTINATION_DIR': fs_str(cache_dir.joinpath('pkgs').as_posix()), + } for source in project.sources: cmd = '{0} download "{1}" -i {2} -d {3}'.format( escape_grouped_arguments(which_pip()), @@ -1506,7 +1521,7 @@ def pip_download(package_name): source['url'], project.download_location, ) - c = delegator.run(cmd) + c = delegator.run(cmd, env=pip_config) if c.return_code == 0: break diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py index bea31585..4205f6e0 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py @@ -10,10 +10,7 @@ import sysconfig import warnings from collections import OrderedDict -try: - import pip._internal.utils.glibc -except ImportError: - import pip.utils.glibc +import pipenv.patched.notpip._internal.utils.glibc logger = logging.getLogger(__name__) @@ -157,7 +154,7 @@ def is_manylinux1_compatible(): pass # Check glibc version. CentOS 5 uses glibc 2.5. - return pip._internal.utils.glibc.have_compatible_glibc(2, 5) + return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 5) def get_darwin_arches(major, minor, machine): diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 6355f959..c435be07 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, import hashlib import os +import sys from contextlib import contextmanager from shutil import rmtree @@ -20,14 +21,17 @@ from .._compat import ( SafeFileCache, ) -from notpip._vendor.packaging.requirements import InvalidRequirement +from notpip._vendor.packaging.requirements import InvalidRequirement, Requirement +from notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from notpip._vendor.packaging.specifiers import SpecifierSet from notpip._vendor.pyparsing import ParseException from ..cache import CACHE_DIR from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound -from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, + make_install_requirement, format_requirement, dedup) + from .base import BaseRepository @@ -159,7 +163,13 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match - all_candidates = self.find_all_candidates(ireq.name) + py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) + all_candidates = [] + for c in self.find_all_candidates(ireq.name): + if c.requires_python and not SpecifierSet(c.requires_python).contains(py_version): + continue + all_candidates.append(c) + candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) try: matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), @@ -188,21 +198,33 @@ class PyPIRepository(BaseRepository): raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq)) def gen(ireq): - if self.DEFAULT_INDEX_URL in self.finder.index_urls: + if self.DEFAULT_INDEX_URL not in self.finder.index_urls: + return - url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) - r = self.session.get(url) + url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) + releases = self.session.get(url).json()['releases'] - # TODO: Latest isn't always latest. - latest = list(r.json()['releases'].keys())[-1] - if str(ireq.req.specifier) == '=={0}'.format(latest): - latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest) - latest_requires = self.session.get(latest_url) - for requires in latest_requires.json().get('info', {}).get('requires_dist', {}): - i = InstallRequirement.from_line(requires) + matches = [ + r for r in releases + if '=={0}'.format(r) == str(ireq.req.specifier) + ] + if not matches: + return - if 'extra' not in repr(i.markers): - yield i + release_requires = self.session.get( + 'https://pypi.org/pypi/{0}/{1}/json'.format( + ireq.req.name, matches[0], + ), + ).json() + try: + requires_dist = release_requires['info']['requires_dist'] + except KeyError: + return + + for requires in requires_dist: + i = InstallRequirement.from_line(requires) + if 'extra' not in repr(i.markers): + yield i try: if ireq not in self._json_dep_cache: @@ -226,7 +248,6 @@ class PyPIRepository(BaseRepository): return json_results - def get_legacy_dependencies(self, ireq): """ Given a pinned or an editable InstallRequirement, returns a set of @@ -245,7 +266,13 @@ class PyPIRepository(BaseRepository): setup_requires = self.finder.get_extras_links( dist.get_metadata_lines('requires.txt') ) - except TypeError: + # HACK: Sometimes the InstallRequirement doesn't properly get + # these values set on it during the resolution process. It's + # difficult to pin down what is going wrong. This fixes things. + ireq.version = dist.version + ireq.project_name = dist.project_name + ireq.req = dist.as_requirement() + except (TypeError, ValueError): pass if ireq not in self._dependencies_cache: diff --git a/pipenv/project.py b/pipenv/project.py index 986b7683..bf86b10c 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -20,7 +20,7 @@ except ImportError: from pathlib2 import Path from .cmdparse import Script -from .vendor.requirementslib import Requirement +from .vendor.requirementslib.requirements import Requirement from .utils import ( atomic_open_for_write, mkdir_p, @@ -728,8 +728,8 @@ class Project(object): # Read and append Pipfile. p = self.parsed_pipfile # Don't re-capitalize file URLs or VCSs. - package = Requirement.from_line(package_name) - converted = first(package.as_pipfile().values()) + package = Requirement.from_line(package_name.strip()) + _, converted = package.pipfile_entry key = 'dev-packages' if dev else 'packages' # Set empty group if it doesn't exist yet. if key not in p: diff --git a/pipenv/utils.py b/pipenv/utils.py index ce95ce52..27ce9d30 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1,20 +1,20 @@ # -*- coding: utf-8 -*- import errno +import logging import os import re -import hashlib -import tempfile -import sys import shutil -import logging +import sys + import crayons import parse import six import stat import warnings -from click import echo as click_echo +from click import echo as click_echo from first import first + try: from weakref import finalize except ImportError: @@ -28,9 +28,10 @@ except ImportError: def detach(self): return False +logging.basicConfig(level=logging.ERROR) + from time import time -logging.basicConfig(level=logging.ERROR) try: from urllib.parse import urlparse except ImportError: @@ -48,7 +49,7 @@ from .pep508checker import lookup from .environments import ( PIPENV_MAX_ROUNDS, PIPENV_CACHE_DIR, - PIPENV_MAX_RETRIES + PIPENV_MAX_RETRIES, ) try: @@ -219,8 +220,8 @@ def prepare_pip_source_args(sources, pip_args=None): def actually_resolve_reps( deps, index_lookup, markers_lookup, project, sources, verbose, clear, pre, req_dir=None ): - from .patched.notpip._internal import basecommand, req - from .patched.notpip._vendor import requests as pip_requests + from .patched.notpip._internal import basecommand + from .patched.notpip._internal.req import parse_requirements from .patched.notpip._internal.exceptions import DistributionNotFound from .patched.notpip._vendor.requests.exceptions import HTTPError from pipenv.patched.piptools.resolver import Resolver @@ -228,7 +229,7 @@ def actually_resolve_reps( from pipenv.patched.piptools.scripts.compile import get_pip_command from pipenv.patched.piptools import logging as piptools_logging from pipenv.patched.piptools.exceptions import NoCandidateFound - from ._compat import TemporaryDirectory + from ._compat import TemporaryDirectory, NamedTemporaryFile class PipCommand(basecommand.Command): """Needed for pip-tools.""" @@ -240,54 +241,51 @@ def actually_resolve_reps( req_dir = TemporaryDirectory(suffix='-requirements', prefix='pipenv-') cleanup_req_dir = True for dep in deps: - if dep: - if dep.startswith('-e '): - constraint = req.InstallRequirement.from_editable( - dep[len('-e '):] - ) - else: - fd, t = tempfile.mkstemp( - prefix='pipenv-', suffix='-requirement.txt', dir=req_dir.name - ) - with os.fdopen(fd, 'w') as f: - f.write(dep) - constraint = [ - c for c in req.parse_requirements(t, session=pip_requests) - ][ - 0 - ] - # extra_constraints = [] - if ' -i ' in dep: - index_lookup[constraint.name] = project.get_source( - url=dep.split(' -i ')[1] - ).get( - 'name' - ) - if constraint.markers: - markers_lookup[constraint.name] = str( - constraint.markers - ).replace( - '"', "'" - ) - constraints.append(constraint) + if not dep: + continue + url = None + if ' -i ' in dep: + dep, url = dep.split(' -i ') + req = Requirement.from_line(dep) + + # req.as_line() is theoratically the same as dep, but is guarenteed to + # be normalized. This is safer than passing in dep. + # TODO: Stop passing dep lines around; just use requirement objects. + constraints.append(req.as_line()) + # extra_constraints = [] + + if url: + index_lookup[req.name] = project.get_source(url=url).get('name') + if req.markers: + markers_lookup[req.name] = req.markers.replace('"', "'") + constraints_file = None pip_command = get_pip_command() pip_args = [] if sources: pip_args = prepare_pip_source_args(sources, pip_args) + with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f: + if sources: + requirementstxt_sources = ' '.join(pip_args).replace(' --', '\n--') + f.write(u'{0}\n'.format(requirementstxt_sources)) + f.write(u'\n'.join([_constraint for _constraint in constraints])) + constraints_file = f.name if verbose: print('Using pip: {0}'.format(' '.join(pip_args))) + pip_args = pip_args.extend(['--cache-dir', PIPENV_CACHE_DIR]) pip_options, _ = pip_command.parse_args(pip_args) session = pip_command._build_session(pip_options) pypi = PyPIRepository( - pip_options=pip_options, use_json=False, session=session + pip_options=pip_options, use_json=True, session=session ) if verbose: logging.log.verbose = True piptools_logging.log.verbose = True resolved_tree = set() - resolver = Resolver( - constraints=constraints, + constraints=parse_requirements( + constraints_file, + finder=pypi.finder, session=pypi.session, options=pip_options, + ), repository=pypi, clear_caches=clear, prereleases=pre, @@ -1130,7 +1128,7 @@ def extract_uri_from_vcs_dep(dep): return None -def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): +def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): target_dir = os.path.join(src_dir, name) target_rev = vcs_obj.make_rev_options(rev) if not os.path.exists(target_dir): @@ -1139,50 +1137,100 @@ def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): return vcs_obj.get_revision(target_dir) -def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False): - from ._compat import vcs - section = 'vcs_dev_packages' if dev else 'vcs_packages' +def get_vcs_deps( + project, + pip_freeze=None, + which=None, + verbose=False, + clear=False, + pre=False, + allow_global=False, + dev=False, +): + from .patched.notpip._internal.vcs import VcsSupport + + section = "vcs_dev_packages" if dev else "vcs_packages" lines = [] lockfiles = [] try: packages = getattr(project, section) except AttributeError: return [], [] - vcs_registry = vcs() + src_dir = Path( + os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src")) + ) + src_dir.mkdir(mode=0o775, exist_ok=True) + vcs_registry = VcsSupport vcs_uri_map = { - extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')} + extract_uri_from_vcs_dep(v): {"name": k, "ref": v.get("ref")} for k, v in packages.items() } - for line in pip_freeze.strip().split('\n'): + for line in pip_freeze.strip().split("\n"): # if the line doesn't match a vcs dependency in the Pipfile, # ignore it _vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line) if not _vcs_match: continue - pipfile_name = vcs_uri_map[_vcs_match]['name'] - pipfile_rev = vcs_uri_map[_vcs_match]['ref'] - src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src')) - mkdir_p(src_dir) + pipfile_name = vcs_uri_map[_vcs_match]["name"] + pipfile_rev = vcs_uri_map[_vcs_match]["ref"] + pipfile_req = Requirement.from_pipfile(pipfile_name, [], packages[pipfile_name]) names = {pipfile_name} - _pip_uri = line.lstrip('-e ') - backend_name = str(_pip_uri.split('+', 1)[0]) - backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)] - __vcs = backend(url=_pip_uri) - + backend = vcs_registry()._registry.get(pipfile_req.vcs) + # TODO: Why doesn't pip freeze list 'git+git://' formatted urls? + if line.startswith("-e ") and not "{0}+".format(pipfile_req.vcs) in line: + line = line.replace("-e ", "-e {0}+".format(pipfile_req.vcs)) installed = Requirement.from_line(line) + __vcs = backend(url=installed.req.uri) + names.add(installed.normalized_name) locked_rev = None for _name in names: - locked_rev = install_or_update_vcs(__vcs, src_dir, _name, rev=pipfile_rev) + locked_rev = install_or_update_vcs( + __vcs, src_dir.as_posix(), _name, rev=pipfile_rev + ) if installed.is_vcs: installed.req.ref = locked_rev lockfiles.append({pipfile_name: installed.pipfile_entry[1]}) - pipfile_srcdir = os.path.join(src_dir, pipfile_name) - lockfile_srcdir = os.path.join(src_dir, installed.normalized_name) + pipfile_srcdir = (src_dir / pipfile_name).as_posix() + lockfile_srcdir = (src_dir / installed.normalized_name).as_posix() lines.append(line) if os.path.exists(pipfile_srcdir): - lockfiles.extend(venv_resolve_deps(['-e {0}'.format(pipfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global)) + lockfiles.extend( + venv_resolve_deps( + ["-e {0}".format(pipfile_srcdir)], + which=which, + verbose=verbose, + project=project, + clear=clear, + pre=pre, + allow_global=allow_global, + ) + ) else: - lockfiles.extend(venv_resolve_deps(['-e {0}'.format(lockfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global)) + lockfiles.extend( + venv_resolve_deps( + ["-e {0}".format(lockfile_srcdir)], + which=which, + verbose=verbose, + project=project, + clear=clear, + pre=pre, + allow_global=allow_global, + ) + ) return lines, lockfiles + + +def fs_str(string): + """Encodes a string into the proper filesystem encoding + + Borrowed from pip-tools + """ + if isinstance(string, str): + return string + assert not isinstance(string, bytes) + return string.encode(_fs_encoding) + + +_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index ef4d46a8..11e09675 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,4 +1,4 @@ # -*- coding=utf-8 -*- -__version__ = "0.0.4" +__version__ = "0.0.7.dev0" from .requirements import Requirement diff --git a/pipenv/vendor/requirementslib/requirements.py b/pipenv/vendor/requirementslib/requirements.py index 04e561c6..4b167ca8 100644 --- a/pipenv/vendor/requirementslib/requirements.py +++ b/pipenv/vendor/requirementslib/requirements.py @@ -8,7 +8,7 @@ import requirements import six from attr import attrs, attrib, Factory, validators import attr -from ._compat import Link, path_to_url, _strip_extras +from ._compat import Link, path_to_url, _strip_extras, InstallRequirement, Wheel from distlib.markers import Evaluator from packaging.markers import Marker, InvalidMarker from packaging.specifiers import SpecifierSet, InvalidSpecifier @@ -30,6 +30,11 @@ try: except ImportError: from pathlib2 import Path +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + HASH_STRING = " --hash={0}" @@ -261,7 +266,7 @@ class NamedRequirement(BaseRequirement): @property def pipfile_part(self): - pipfile_dict = attr.asdict(self, filter=_filter_none) + pipfile_dict = attr.asdict(self, filter=_filter_none).copy() if "version" not in pipfile_dict: pipfile_dict["version"] = "*" name = pipfile_dict.pop("name") @@ -301,20 +306,23 @@ class FileRequirement(BaseRequirement): @link.default def get_link(self): target = "{0}#egg={1}".format(self.uri, self.name) - return Link(target) + link = Link(target) + if link.is_wheel and self._has_hashed_name: + self.name = os.path.basename(Wheel(link.path).name) + return link @req.default def get_requirement(self): - base = "{0}".format(self.link) - req = first(requirements.parse(base)) + prefix = "-e " if self.editable else "" + line = "{0}{1}".format(prefix, self.link.url) + req = first(requirements.parse(line)) + if self.path and self.link and self.link.scheme.startswith("file"): + req.local_file = True + req.path = self.path + req.uri = None + self._uri_scheme = "file" if self.editable: req.editable = True - if self.link and self.link.scheme.startswith("file"): - if self.path: - req.path = self.path - req.local_file = True - self._uri_scheme = "file" - req.uri = None req.link = self.link return req @@ -338,15 +346,24 @@ class FileRequirement(BaseRequirement): "Supplied requirement is not installable: {0!r}".format(line) ) - if is_valid_url(line): + if is_valid_url(line) and not is_installable_file(line): link = Link(line) else: - _path = Path(line) - link = Link(_path.absolute().as_uri()) - if _path.is_absolute() or _path.as_posix() == ".": - path = _path.as_posix() + if is_valid_url(line): + parsed = urlparse(line) + link = Link('{0}'.format(line)) + if parsed.scheme == "file": + path = Path(parsed.path).absolute().as_posix() + if get_converted_relative_path(path) == ".": + path = "." + line = path else: - path = get_converted_relative_path(line) + _path = Path(line) + link = Link(_path.absolute().as_uri()) + if _path.is_absolute() or _path.as_posix() == ".": + path = _path.as_posix() + else: + path = get_converted_relative_path(line) arg_dict = { "path": path, "uri": link.url_without_fragment, @@ -571,6 +588,7 @@ class Requirement(object): editable = attrib(default=None) hashes = attrib(default=Factory(list), converter=list) extras = attrib(default=Factory(list)) + _ireq = None _INCLUDE_FIELDS = ("name", "markers", "index", "editable", "hashes", "extras") @name.default @@ -749,6 +767,17 @@ class Requirement(object): def pipfile_entry(self): return self.as_pipfile().copy().popitem() + @property + def ireq(self): + if not self._ireq: + ireq_line = self.as_line() + if ireq_line.startswith("-e "): + ireq_line = ireq_line[len("-e "):] + self._ireq = InstallRequirement.from_editable(ireq_line) + else: + self._ireq = InstallRequirement.from_line(ireq_line) + return self._ireq + def _extras_to_string(extras): """Turn a list of extras into a string""" diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index b9358a35..5302630a 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -35,7 +35,10 @@ def is_vcs(pipfile_entry): def get_converted_relative_path(path, relative_to=os.curdir): """Given a vague relative path, return the path relative to the given location""" - return os.path.join(".", os.path.relpath(path, start=relative_to)) + relpath = os.path.relpath(path, start=relative_to) + if os.name == 'nt': + return os.altsep.join([".", relpath]) + return os.path.join(".", relpath) def multi_split(s, split): @@ -73,6 +76,10 @@ def is_installable_file(path): else: return False + parsed = urlparse(path) + if parsed.scheme == 'file': + path = parsed.path + if not os.path.exists(os.path.abspath(path)): return False @@ -90,7 +97,7 @@ def is_installable_file(path): def is_valid_url(url): """Checks if a given string is an url""" pieces = urlparse(url) - return all([pieces.scheme, pieces.netloc]) + return all([pieces.scheme, any([pieces.netloc, pieces.path])]) def pep423_name(name): diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 9bd14dee..4b64b818 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -468,7 +468,10 @@ def generate_patch(ctx, package_path, patch_description, base='HEAD'): pkg = Path(package_path) if len(pkg.parts) != 2 or pkg.parts[0] not in ('vendor', 'patched'): raise ValueError('example usage: generate-patch patched/pew some-description') - patch_fn = '{0}-{1}.patch'.format(pkg.parts[1], patch_description) + if patch_description: + patch_fn = '{0}-{1}.patch'.format(pkg.parts[1], patch_description) + else: + patch_fn = '{0}.patch'.format(pkg.parts[1]) command = 'git diff {base} -p {root} > {out}'.format( base=base, root=Path('pipenv').joinpath(pkg), diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch index 70ccd170..ebebe17e 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch @@ -1,16 +1,25 @@ diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py -index bea31585..9e9609f3 100644 +index bea31585..4205f6e0 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py -@@ -11,9 +11,9 @@ import warnings +@@ -10,10 +10,7 @@ import sysconfig + import warnings from collections import OrderedDict - try: +-try: - import pip._internal.utils.glibc -+ import pipenv.patched.notpip._internal.utils.glibc - except ImportError: +-except ImportError: - import pip.utils.glibc -+ import pipenv.patched.notpip.utils.glibc ++import pipenv.patched.notpip._internal.utils.glibc logger = logging.getLogger(__name__) +@@ -157,7 +154,7 @@ def is_manylinux1_compatible(): + pass + + # Check glibc version. CentOS 5 uses glibc 2.5. +- return pip._internal.utils.glibc.have_compatible_glibc(2, 5) ++ return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 5) + + + def get_darwin_arches(major, minor, machine): diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 6beaad8e..fe2c46d8 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,10 +19,18 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..8320e14 100644 +index 1c4b943..aae0122 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -15,10 +15,16 @@ from .._compat import ( +@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, + + import hashlib + import os ++import sys + from contextlib import contextmanager + from shutil import rmtree + +@@ -15,13 +16,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -32,15 +40,23 @@ index 1c4b943..8320e14 100644 + SafeFileCache, ) -+from pip._vendor.packaging.requirements import InvalidRequirement ++from pip._vendor.packaging.requirements import InvalidRequirement, Requirement ++from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version ++from pip._vendor.packaging.specifiers import SpecifierSet +from pip._vendor.pyparsing import ParseException + from ..cache import CACHE_DIR +from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound - from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) -@@ -37,6 +43,40 @@ except ImportError: +-from ..utils import (fs_str, is_pinned_requirement, lookup_table, +- make_install_requirement) ++from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, ++ make_install_requirement, format_requirement, dedup) ++ + from .base import BaseRepository + + +@@ -37,6 +47,40 @@ except ImportError: from pip.wheel import WheelCache @@ -81,7 +97,7 @@ index 1c4b943..8320e14 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,10 +86,11 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +90,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -95,7 +111,7 @@ index 1c4b943..8320e14 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -74,11 +115,15 @@ class PyPIRepository(BaseRepository): +@@ -74,11 +119,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -113,9 +129,18 @@ index 1c4b943..8320e14 100644 def freshen_build_caches(self): """ -@@ -116,8 +161,11 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +163,19 @@ class PyPIRepository(BaseRepository): + if ireq.editable: + return ireq # return itself as the best match - all_candidates = self.find_all_candidates(ireq.name) +- all_candidates = self.find_all_candidates(ireq.name) ++ py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) ++ all_candidates = [] ++ for c in self.find_all_candidates(ireq.name): ++ if c.requires_python and not SpecifierSet(c.requires_python).contains(py_version): ++ continue ++ all_candidates.append(c) ++ candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) - matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), + try: @@ -126,7 +151,7 @@ index 1c4b943..8320e14 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +174,60 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +184,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -147,21 +172,33 @@ index 1c4b943..8320e14 100644 + raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq)) + + def gen(ireq): -+ if self.DEFAULT_INDEX_URL in self.finder.index_urls: ++ if self.DEFAULT_INDEX_URL not in self.finder.index_urls: ++ return + -+ url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) -+ r = self.session.get(url) ++ url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) ++ releases = self.session.get(url).json()['releases'] + -+ # TODO: Latest isn't always latest. -+ latest = list(r.json()['releases'].keys())[-1] -+ if str(ireq.req.specifier) == '=={0}'.format(latest): -+ latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest) -+ latest_requires = self.session.get(latest_url) -+ for requires in latest_requires.json().get('info', {}).get('requires_dist', {}): -+ i = InstallRequirement.from_line(requires) ++ matches = [ ++ r for r in releases ++ if '=={0}'.format(r) == str(ireq.req.specifier) ++ ] ++ if not matches: ++ return + -+ if 'extra' not in repr(i.markers): -+ yield i ++ release_requires = self.session.get( ++ 'https://pypi.org/pypi/{0}/{1}/json'.format( ++ ireq.req.name, matches[0], ++ ), ++ ).json() ++ try: ++ requires_dist = release_requires['info']['requires_dist'] ++ except KeyError: ++ return ++ ++ for requires in requires_dist: ++ i = InstallRequirement.from_line(requires) ++ if 'extra' not in repr(i.markers): ++ yield i + + try: + if ireq not in self._json_dep_cache: @@ -185,12 +222,11 @@ index 1c4b943..8320e14 100644 + + return json_results + -+ + def get_legacy_dependencies(self, ireq): """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -139,6 +236,18 @@ class PyPIRepository(BaseRepository): +@@ -139,6 +257,24 @@ class PyPIRepository(BaseRepository): if not (ireq.editable or is_pinned_requirement(ireq)): raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq)) @@ -203,13 +239,19 @@ index 1c4b943..8320e14 100644 + setup_requires = self.finder.get_extras_links( + dist.get_metadata_lines('requires.txt') + ) -+ except TypeError: ++ # HACK: Sometimes the InstallRequirement doesn't properly get ++ # these values set on it during the resolution process. It's ++ # difficult to pin down what is going wrong. This fixes things. ++ ireq.version = dist.version ++ ireq.project_name = dist.project_name ++ ireq.req = dist.as_requirement() ++ except (TypeError, ValueError): + pass + if ireq not in self._dependencies_cache: if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # No download_dir for locally available editable requirements. -@@ -164,11 +273,14 @@ class PyPIRepository(BaseRepository): +@@ -164,11 +300,14 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, @@ -226,7 +268,7 @@ index 1c4b943..8320e14 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -190,14 +302,44 @@ class PyPIRepository(BaseRepository): +@@ -190,14 +329,44 @@ class PyPIRepository(BaseRepository): upgrade_strategy="to-satisfy-only", force_reinstall=False, ignore_dependencies=False, @@ -273,7 +315,7 @@ index 1c4b943..8320e14 100644 reqset.cleanup_files() return set(self._dependencies_cache[ireq]) -@@ -224,17 +366,10 @@ class PyPIRepository(BaseRepository): +@@ -224,17 +393,10 @@ class PyPIRepository(BaseRepository): matching_candidates = candidates_by_version[matching_versions[0]] return {