From 513bdfb0d29890d940c3e18d4566bf8599a69636 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 26 May 2018 17:47:28 -0400 Subject: [PATCH 01/19] Update requirementslib to fix windows paths - Fixes #2256 Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/__init__.py | 2 +- pipenv/vendor/requirementslib/requirements.py | 58 ++++++++++++++----- pipenv/vendor/requirementslib/utils.py | 11 +++- 3 files changed, 52 insertions(+), 19 deletions(-) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index ef4d46a8..6727a81e 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,4 +1,4 @@ # -*- coding=utf-8 -*- -__version__ = "0.0.4" +__version__ = "0.0.6" from .requirements import Requirement diff --git a/pipenv/vendor/requirementslib/requirements.py b/pipenv/vendor/requirementslib/requirements.py index 04e561c6..e5989474 100644 --- a/pipenv/vendor/requirementslib/requirements.py +++ b/pipenv/vendor/requirementslib/requirements.py @@ -8,7 +8,7 @@ import requirements import six from attr import attrs, attrib, Factory, validators import attr -from ._compat import Link, path_to_url, _strip_extras +from ._compat import Link, path_to_url, _strip_extras, InstallRequirement from distlib.markers import Evaluator from packaging.markers import Marker, InvalidMarker from packaging.specifiers import SpecifierSet, InvalidSpecifier @@ -30,6 +30,11 @@ try: except ImportError: from pathlib2 import Path +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + HASH_STRING = " --hash={0}" @@ -261,7 +266,7 @@ class NamedRequirement(BaseRequirement): @property def pipfile_part(self): - pipfile_dict = attr.asdict(self, filter=_filter_none) + pipfile_dict = attr.asdict(self, filter=_filter_none).copy() if "version" not in pipfile_dict: pipfile_dict["version"] = "*" name = pipfile_dict.pop("name") @@ -305,16 +310,16 @@ class FileRequirement(BaseRequirement): @req.default def get_requirement(self): - base = "{0}".format(self.link) - req = first(requirements.parse(base)) + prefix = "-e " if self.editable else "" + line = "{0}{1}".format(prefix, self.link.url) + req = first(requirements.parse(line)) + if self.path and self.link and self.link.scheme.startswith("file"): + req.local_file = True + req.path = self.path + req.uri = None + self._uri_scheme = "file" if self.editable: req.editable = True - if self.link and self.link.scheme.startswith("file"): - if self.path: - req.path = self.path - req.local_file = True - self._uri_scheme = "file" - req.uri = None req.link = self.link return req @@ -338,15 +343,24 @@ class FileRequirement(BaseRequirement): "Supplied requirement is not installable: {0!r}".format(line) ) - if is_valid_url(line): + if is_valid_url(line) and not is_installable_file(line): link = Link(line) else: - _path = Path(line) - link = Link(_path.absolute().as_uri()) - if _path.is_absolute() or _path.as_posix() == ".": - path = _path.as_posix() + if is_valid_url(line): + parsed = urlparse(line) + link = Link('{0}'.format(line)) + if parsed.scheme == "file": + path = Path(parsed.path).absolute().as_posix() + if get_converted_relative_path(path) == ".": + path = "." + line = path else: - path = get_converted_relative_path(line) + _path = Path(line) + link = Link(_path.absolute().as_uri()) + if _path.is_absolute() or _path.as_posix() == ".": + path = _path.as_posix() + else: + path = get_converted_relative_path(line) arg_dict = { "path": path, "uri": link.url_without_fragment, @@ -571,6 +585,7 @@ class Requirement(object): editable = attrib(default=None) hashes = attrib(default=Factory(list), converter=list) extras = attrib(default=Factory(list)) + _ireq = None _INCLUDE_FIELDS = ("name", "markers", "index", "editable", "hashes", "extras") @name.default @@ -749,6 +764,17 @@ class Requirement(object): def pipfile_entry(self): return self.as_pipfile().copy().popitem() + @property + def ireq(self): + if not self._ireq: + ireq_line = self.as_line() + if ireq_line.startswith("-e "): + ireq_line = ireq_line[len("-e "):] + self._ireq = InstallRequirement.from_editable(ireq_line) + else: + self._ireq = InstallRequirement.from_line(ireq_line) + return self._ireq + def _extras_to_string(extras): """Turn a list of extras into a string""" diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index b9358a35..5302630a 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -35,7 +35,10 @@ def is_vcs(pipfile_entry): def get_converted_relative_path(path, relative_to=os.curdir): """Given a vague relative path, return the path relative to the given location""" - return os.path.join(".", os.path.relpath(path, start=relative_to)) + relpath = os.path.relpath(path, start=relative_to) + if os.name == 'nt': + return os.altsep.join([".", relpath]) + return os.path.join(".", relpath) def multi_split(s, split): @@ -73,6 +76,10 @@ def is_installable_file(path): else: return False + parsed = urlparse(path) + if parsed.scheme == 'file': + path = parsed.path + if not os.path.exists(os.path.abspath(path)): return False @@ -90,7 +97,7 @@ def is_installable_file(path): def is_valid_url(url): """Checks if a given string is an url""" pieces = urlparse(url) - return all([pieces.scheme, pieces.netloc]) + return all([pieces.scheme, any([pieces.netloc, pieces.path])]) def pep423_name(name): From d55e400379f67d4cb2baaf8befb2c0ec8a24fe72 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 26 May 2018 19:57:41 -0400 Subject: [PATCH 02/19] Better handling of wheels / update requirementslib Signed-off-by: Dan Ryan --- pipenv/patched/notpip/_internal/pep425tags.py | 7 ++----- pipenv/vendor/requirementslib/__init__.py | 2 +- pipenv/vendor/requirementslib/requirements.py | 7 +++++-- .../patched/_post-pip-update-pep425tags.patch | 21 +++++++++++++------ 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py index bea31585..4205f6e0 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py @@ -10,10 +10,7 @@ import sysconfig import warnings from collections import OrderedDict -try: - import pip._internal.utils.glibc -except ImportError: - import pip.utils.glibc +import pipenv.patched.notpip._internal.utils.glibc logger = logging.getLogger(__name__) @@ -157,7 +154,7 @@ def is_manylinux1_compatible(): pass # Check glibc version. CentOS 5 uses glibc 2.5. - return pip._internal.utils.glibc.have_compatible_glibc(2, 5) + return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 5) def get_darwin_arches(major, minor, machine): diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 6727a81e..11e09675 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,4 +1,4 @@ # -*- coding=utf-8 -*- -__version__ = "0.0.6" +__version__ = "0.0.7.dev0" from .requirements import Requirement diff --git a/pipenv/vendor/requirementslib/requirements.py b/pipenv/vendor/requirementslib/requirements.py index e5989474..4b167ca8 100644 --- a/pipenv/vendor/requirementslib/requirements.py +++ b/pipenv/vendor/requirementslib/requirements.py @@ -8,7 +8,7 @@ import requirements import six from attr import attrs, attrib, Factory, validators import attr -from ._compat import Link, path_to_url, _strip_extras, InstallRequirement +from ._compat import Link, path_to_url, _strip_extras, InstallRequirement, Wheel from distlib.markers import Evaluator from packaging.markers import Marker, InvalidMarker from packaging.specifiers import SpecifierSet, InvalidSpecifier @@ -306,7 +306,10 @@ class FileRequirement(BaseRequirement): @link.default def get_link(self): target = "{0}#egg={1}".format(self.uri, self.name) - return Link(target) + link = Link(target) + if link.is_wheel and self._has_hashed_name: + self.name = os.path.basename(Wheel(link.path).name) + return link @req.default def get_requirement(self): diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch index 70ccd170..ebebe17e 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch @@ -1,16 +1,25 @@ diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py -index bea31585..9e9609f3 100644 +index bea31585..4205f6e0 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py -@@ -11,9 +11,9 @@ import warnings +@@ -10,10 +10,7 @@ import sysconfig + import warnings from collections import OrderedDict - try: +-try: - import pip._internal.utils.glibc -+ import pipenv.patched.notpip._internal.utils.glibc - except ImportError: +-except ImportError: - import pip.utils.glibc -+ import pipenv.patched.notpip.utils.glibc ++import pipenv.patched.notpip._internal.utils.glibc logger = logging.getLogger(__name__) +@@ -157,7 +154,7 @@ def is_manylinux1_compatible(): + pass + + # Check glibc version. CentOS 5 uses glibc 2.5. +- return pip._internal.utils.glibc.have_compatible_glibc(2, 5) ++ return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 5) + + + def get_darwin_arches(major, minor, machine): From 7abc2fd3a1fa371538c37f764dfcb75debb098f4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 24 May 2018 22:06:29 -0400 Subject: [PATCH 03/19] Use piptools constraints files more properly Signed-off-by: Dan Ryan --- pipenv/utils.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index ce95ce52..01cf6001 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -219,7 +219,8 @@ def prepare_pip_source_args(sources, pip_args=None): def actually_resolve_reps( deps, index_lookup, markers_lookup, project, sources, verbose, clear, pre, req_dir=None ): - from .patched.notpip._internal import basecommand, req + from .patched.notpip._internal import basecommand + from .patched.notpip._internal.req import parse_requirements from .patched.notpip._vendor import requests as pip_requests from .patched.notpip._internal.exceptions import DistributionNotFound from .patched.notpip._vendor.requests.exceptions import HTTPError @@ -228,13 +229,14 @@ def actually_resolve_reps( from pipenv.patched.piptools.scripts.compile import get_pip_command from pipenv.patched.piptools import logging as piptools_logging from pipenv.patched.piptools.exceptions import NoCandidateFound - from ._compat import TemporaryDirectory + from ._compat import TemporaryDirectory, NamedTemporaryFile class PipCommand(basecommand.Command): """Needed for pip-tools.""" name = 'PipCommand' constraints = [] + tmpfile_constraints = [] cleanup_req_dir = False if not req_dir: req_dir = TemporaryDirectory(suffix='-requirements', prefix='pipenv-') @@ -246,30 +248,26 @@ def actually_resolve_reps( dep[len('-e '):] ) else: - fd, t = tempfile.mkstemp( - prefix='pipenv-', suffix='-requirement.txt', dir=req_dir.name - ) - with os.fdopen(fd, 'w') as f: - f.write(dep) - constraint = [ - c for c in req.parse_requirements(t, session=pip_requests) - ][ - 0 - ] + tmpfile_constraints.append(dep) + req = Requirement.from_line(dep) # extra_constraints = [] if ' -i ' in dep: - index_lookup[constraint.name] = project.get_source( + index_lookup[req.name] = project.get_source( url=dep.split(' -i ')[1] ).get( 'name' ) - if constraint.markers: - markers_lookup[constraint.name] = str( - constraint.markers + if dep.markers: + markers_lookup[dep.name] = str( + dep.markers_as_pip ).replace( '"', "'" ) - constraints.append(constraint) + constraints.append(req) + constraints_file = None + with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f: + f.write('\n'.join(tmpfile_constraints)) + constraints_file = f.name pip_command = get_pip_command() pip_args = [] if sources: @@ -285,9 +283,9 @@ def actually_resolve_reps( logging.log.verbose = True piptools_logging.log.verbose = True resolved_tree = set() - + piptools_constraints = [c for c in parse_requirements(constraints_file, finder=pypi.finder, session=pypi.session, options=pip_options)] resolver = Resolver( - constraints=constraints, + constraints=piptools_constraints, repository=pypi, clear_caches=clear, prereleases=pre, From bba2f38d91b0eba9c752b3b4517badba9b14bdc6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 25 May 2018 20:20:55 -0400 Subject: [PATCH 04/19] Patch piptools to use current environment python - Fixes #2088, #2234, #1901 - Fully leverage piptools' compile functionality by using constraints in the same `RequirementSet` during resolution - Use `PIP_PYTHON_PATH` for compatibility check to filter out `requires_python` markers - Fix vcs resolution - Update JSON API endpoints - Enhance resolution for editable dependencies - Minor fix for adding packages to pipfiles Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 36 ++++++--- pipenv/project.py | 6 +- pipenv/utils.py | 56 ++++++-------- .../vendoring/patches/patched/piptools.patch | 73 +++++++++++++------ 4 files changed, 105 insertions(+), 66 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 6355f959..ea709a1b 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, import hashlib import os +import sys from contextlib import contextmanager from shutil import rmtree @@ -20,14 +21,17 @@ from .._compat import ( SafeFileCache, ) -from notpip._vendor.packaging.requirements import InvalidRequirement +from notpip._vendor.packaging.requirements import InvalidRequirement, Requirement +from notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from notpip._vendor.packaging.specifiers import SpecifierSet from notpip._vendor.pyparsing import ParseException from ..cache import CACHE_DIR from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound -from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, + make_install_requirement, format_requirement, dedup) + from .base import BaseRepository @@ -159,7 +163,15 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match - all_candidates = self.find_all_candidates(ireq.name) + _all_candidates = self.find_all_candidates(ireq.name) + all_candidates = [] + py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) + for c in _all_candidates: + if c.requires_python: + python_specifier = SpecifierSet(c.requires_python) + if not python_specifier.contains(py_version): + continue + all_candidates.append(c) candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) try: matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), @@ -194,11 +206,12 @@ class PyPIRepository(BaseRepository): r = self.session.get(url) # TODO: Latest isn't always latest. - latest = list(r.json()['releases'].keys())[-1] - if str(ireq.req.specifier) == '=={0}'.format(latest): - latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest) - latest_requires = self.session.get(latest_url) - for requires in latest_requires.json().get('info', {}).get('requires_dist', {}): + releases = list(r.json()['releases'].keys()) + match = [r for r in releases if '=={0}'.format(r) == str(ireq.req.specifier)] + if match: + release_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, match[0]) + release_requires = self.session.get(release_url) + for requires in release_requires.json().get('info', {}).get('requires_dist', {}): i = InstallRequirement.from_line(requires) if 'extra' not in repr(i.markers): @@ -245,7 +258,10 @@ class PyPIRepository(BaseRepository): setup_requires = self.finder.get_extras_links( dist.get_metadata_lines('requires.txt') ) - except TypeError: + ireq.version = dist.version + ireq.project_name = dist.project_name + ireq.req = dist.as_requirement() + except (TypeError, ValueError): pass if ireq not in self._dependencies_cache: diff --git a/pipenv/project.py b/pipenv/project.py index 986b7683..bf86b10c 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -20,7 +20,7 @@ except ImportError: from pathlib2 import Path from .cmdparse import Script -from .vendor.requirementslib import Requirement +from .vendor.requirementslib.requirements import Requirement from .utils import ( atomic_open_for_write, mkdir_p, @@ -728,8 +728,8 @@ class Project(object): # Read and append Pipfile. p = self.parsed_pipfile # Don't re-capitalize file URLs or VCSs. - package = Requirement.from_line(package_name) - converted = first(package.as_pipfile().values()) + package = Requirement.from_line(package_name.strip()) + _, converted = package.pipfile_entry key = 'dev-packages' if dev else 'packages' # Set empty group if it doesn't exist yet. if key not in p: diff --git a/pipenv/utils.py b/pipenv/utils.py index 01cf6001..e0a8070b 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -221,6 +221,7 @@ def actually_resolve_reps( ): from .patched.notpip._internal import basecommand from .patched.notpip._internal.req import parse_requirements + from .patched.notpip._internal.req.req_install import InstallRequirement from .patched.notpip._vendor import requests as pip_requests from .patched.notpip._internal.exceptions import DistributionNotFound from .patched.notpip._vendor.requests.exceptions import HTTPError @@ -236,48 +237,37 @@ def actually_resolve_reps( name = 'PipCommand' constraints = [] - tmpfile_constraints = [] cleanup_req_dir = False if not req_dir: req_dir = TemporaryDirectory(suffix='-requirements', prefix='pipenv-') cleanup_req_dir = True for dep in deps: if dep: - if dep.startswith('-e '): - constraint = req.InstallRequirement.from_editable( - dep[len('-e '):] - ) - else: - tmpfile_constraints.append(dep) - req = Requirement.from_line(dep) - # extra_constraints = [] + url = None if ' -i ' in dep: - index_lookup[req.name] = project.get_source( - url=dep.split(' -i ')[1] - ).get( - 'name' - ) - if dep.markers: - markers_lookup[dep.name] = str( - dep.markers_as_pip - ).replace( - '"', "'" - ) - constraints.append(req) + dep, url = dep.split(' -i ') + req = Requirement.from_line(dep) + _line = req.as_line() + constraints.append(_line) + # extra_constraints = [] + if url: + index_lookup[req.name] = project.get_source(url=url).get('name') + if req.markers: + markers_lookup[req.name] = req.markers_as_pip constraints_file = None - with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f: - f.write('\n'.join(tmpfile_constraints)) - constraints_file = f.name pip_command = get_pip_command() pip_args = [] if sources: pip_args = prepare_pip_source_args(sources, pip_args) + with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f: + f.write(u'\n'.join([_constraint for _constraint in constraints])) + constraints_file = f.name if verbose: print('Using pip: {0}'.format(' '.join(pip_args))) pip_options, _ = pip_command.parse_args(pip_args) session = pip_command._build_session(pip_options) pypi = PyPIRepository( - pip_options=pip_options, use_json=False, session=session + pip_options=pip_options, use_json=True, session=session ) if verbose: logging.log.verbose = True @@ -1138,7 +1128,7 @@ def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False): - from ._compat import vcs + from .patched.notpip._internal.vcs import VcsSupport section = 'vcs_dev_packages' if dev else 'vcs_packages' lines = [] lockfiles = [] @@ -1146,7 +1136,7 @@ def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=Fals packages = getattr(project, section) except AttributeError: return [], [] - vcs_registry = vcs() + vcs_registry = VcsSupport vcs_uri_map = { extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')} for k, v in packages.items() @@ -1162,13 +1152,15 @@ def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=Fals pipfile_rev = vcs_uri_map[_vcs_match]['ref'] src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src')) mkdir_p(src_dir) + pipfile_req = Requirement.from_pipfile(pipfile_name, [], packages[pipfile_name]) names = {pipfile_name} - _pip_uri = line.lstrip('-e ') - backend_name = str(_pip_uri.split('+', 1)[0]) - backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)] - __vcs = backend(url=_pip_uri) - + backend = vcs_registry()._registry.get(pipfile_req.vcs) + # TODO: Why doesn't pip freeze list 'git+git://' formatted urls? + if line.startswith('-e ') and not '{0}+'.format(pipfile_req.vcs) in line: + line = line.replace('-e ', '-e {0}+'.format(pipfile_req.vcs)) installed = Requirement.from_line(line) + __vcs = backend(url=installed.req.uri) + names.add(installed.normalized_name) locked_rev = None for _name in names: diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 6beaad8e..a3220d39 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,10 +19,18 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..8320e14 100644 +index 1c4b943..858d697 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -15,10 +15,16 @@ from .._compat import ( +@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, + + import hashlib + import os ++import sys + from contextlib import contextmanager + from shutil import rmtree + +@@ -15,13 +16,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -32,15 +40,23 @@ index 1c4b943..8320e14 100644 + SafeFileCache, ) -+from pip._vendor.packaging.requirements import InvalidRequirement ++from pip._vendor.packaging.requirements import InvalidRequirement, Requirement ++from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version ++from pip._vendor.packaging.specifiers import SpecifierSet +from pip._vendor.pyparsing import ParseException + from ..cache import CACHE_DIR +from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound - from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) -@@ -37,6 +43,40 @@ except ImportError: +-from ..utils import (fs_str, is_pinned_requirement, lookup_table, +- make_install_requirement) ++from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, ++ make_install_requirement, format_requirement, dedup) ++ + from .base import BaseRepository + + +@@ -37,6 +47,40 @@ except ImportError: from pip.wheel import WheelCache @@ -81,7 +97,7 @@ index 1c4b943..8320e14 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,10 +86,11 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +90,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -95,7 +111,7 @@ index 1c4b943..8320e14 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -74,11 +115,15 @@ class PyPIRepository(BaseRepository): +@@ -74,11 +119,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -113,9 +129,20 @@ index 1c4b943..8320e14 100644 def freshen_build_caches(self): """ -@@ -116,8 +161,11 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +163,21 @@ class PyPIRepository(BaseRepository): + if ireq.editable: + return ireq # return itself as the best match - all_candidates = self.find_all_candidates(ireq.name) +- all_candidates = self.find_all_candidates(ireq.name) ++ _all_candidates = self.find_all_candidates(ireq.name) ++ all_candidates = [] ++ py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) ++ for c in _all_candidates: ++ if c.requires_python: ++ python_specifier = SpecifierSet(c.requires_python) ++ if not python_specifier.contains(py_version): ++ continue ++ all_candidates.append(c) candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) - matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), + try: @@ -126,7 +153,7 @@ index 1c4b943..8320e14 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +174,60 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +186,61 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -153,11 +180,12 @@ index 1c4b943..8320e14 100644 + r = self.session.get(url) + + # TODO: Latest isn't always latest. -+ latest = list(r.json()['releases'].keys())[-1] -+ if str(ireq.req.specifier) == '=={0}'.format(latest): -+ latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest) -+ latest_requires = self.session.get(latest_url) -+ for requires in latest_requires.json().get('info', {}).get('requires_dist', {}): ++ releases = list(r.json()['releases'].keys()) ++ match = [r for r in releases if '=={0}'.format(r) == str(ireq.req.specifier)] ++ if match: ++ release_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, match[0]) ++ release_requires = self.session.get(release_url) ++ for requires in release_requires.json().get('info', {}).get('requires_dist', {}): + i = InstallRequirement.from_line(requires) + + if 'extra' not in repr(i.markers): @@ -190,7 +218,7 @@ index 1c4b943..8320e14 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -139,6 +236,18 @@ class PyPIRepository(BaseRepository): +@@ -139,6 +249,21 @@ class PyPIRepository(BaseRepository): if not (ireq.editable or is_pinned_requirement(ireq)): raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq)) @@ -203,13 +231,16 @@ index 1c4b943..8320e14 100644 + setup_requires = self.finder.get_extras_links( + dist.get_metadata_lines('requires.txt') + ) -+ except TypeError: ++ ireq.version = dist.version ++ ireq.project_name = dist.project_name ++ ireq.req = dist.as_requirement() ++ except (TypeError, ValueError): + pass + if ireq not in self._dependencies_cache: if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # No download_dir for locally available editable requirements. -@@ -164,11 +273,14 @@ class PyPIRepository(BaseRepository): +@@ -164,11 +289,14 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, @@ -226,7 +257,7 @@ index 1c4b943..8320e14 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -190,14 +302,44 @@ class PyPIRepository(BaseRepository): +@@ -190,14 +318,44 @@ class PyPIRepository(BaseRepository): upgrade_strategy="to-satisfy-only", force_reinstall=False, ignore_dependencies=False, @@ -273,7 +304,7 @@ index 1c4b943..8320e14 100644 reqset.cleanup_files() return set(self._dependencies_cache[ireq]) -@@ -224,17 +366,10 @@ class PyPIRepository(BaseRepository): +@@ -224,17 +382,10 @@ class PyPIRepository(BaseRepository): matching_candidates = candidates_by_version[matching_versions[0]] return { From 84c0b0919329be4299d3973d9e77b98547368087 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 26 May 2018 15:23:32 -0400 Subject: [PATCH 05/19] Fix marker parsing bug Signed-off-by: Dan Ryan --- pipenv/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index e0a8070b..3237b24c 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -253,7 +253,7 @@ def actually_resolve_reps( if url: index_lookup[req.name] = project.get_source(url=url).get('name') if req.markers: - markers_lookup[req.name] = req.markers_as_pip + markers_lookup[req.name] = req.markers.replace('"', "'") constraints_file = None pip_command = get_pip_command() pip_args = [] From f532b2eb864a4012179d024a1c5228cccb16af18 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 26 May 2018 23:39:34 -0400 Subject: [PATCH 06/19] Use pipenv cache dir for caching wheels Signed-off-by: Dan Ryan --- pipenv/core.py | 5 +++-- pipenv/utils.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 75bc6276..0f5c7e4f 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -73,6 +73,7 @@ from .environments import ( PIPENV_SHELL, PIPENV_PYTHON, PIPENV_VIRTUALENV, + PIPENV_CACHE_DIR, ) # Backport required for earlier versions of Python. @@ -1494,7 +1495,7 @@ def pip_install( ) if verbose: click.echo('$ {0}'.format(pip_command), err=True) - c = delegator.run(pip_command, block=block) + c = delegator.run(pip_command, block=block, env={'PIP_CACHE_DIR': PIPENV_CACHE_DIR}) return c @@ -1506,7 +1507,7 @@ def pip_download(package_name): source['url'], project.download_location, ) - c = delegator.run(cmd) + c = delegator.run(cmd, env={'PIP_CACHE_DIR': PIPENV_CACHE_DIR}) if c.return_code == 0: break diff --git a/pipenv/utils.py b/pipenv/utils.py index 3237b24c..599d8a0c 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -264,6 +264,7 @@ def actually_resolve_reps( constraints_file = f.name if verbose: print('Using pip: {0}'.format(' '.join(pip_args))) + pip_args = pip_args.extend(['--cache-dir', PIPENV_CACHE_DIR]) pip_options, _ = pip_command.parse_args(pip_args) session = pip_command._build_session(pip_options) pypi = PyPIRepository( From f91371aa559a46d953ac0731db0542db5ddd1343 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sun, 27 May 2018 14:02:00 +0900 Subject: [PATCH 07/19] Code clean up This shouldn't affect any functionalities, only clean up some generator usages to reduce repeated looping through requirements. --- pipenv/patched/piptools/repositories/pypi.py | 51 +++++++++++-------- pipenv/utils.py | 53 +++++++++++--------- 2 files changed, 57 insertions(+), 47 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index ea709a1b..43b463af 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -163,15 +163,12 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match - _all_candidates = self.find_all_candidates(ireq.name) - all_candidates = [] py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) - for c in _all_candidates: - if c.requires_python: - python_specifier = SpecifierSet(c.requires_python) - if not python_specifier.contains(py_version): - continue - all_candidates.append(c) + all_candidates = [ + c for c in self.find_all_candidates(ireq.name) + if SpecifierSet(c.requires_python).contains(py_version) + ] + candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) try: matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), @@ -200,22 +197,33 @@ class PyPIRepository(BaseRepository): raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq)) def gen(ireq): - if self.DEFAULT_INDEX_URL in self.finder.index_urls: + if self.DEFAULT_INDEX_URL not in self.finder.index_urls: + return - url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) - r = self.session.get(url) + url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) + releases = self.session.get(url).json()['releases'] - # TODO: Latest isn't always latest. - releases = list(r.json()['releases'].keys()) - match = [r for r in releases if '=={0}'.format(r) == str(ireq.req.specifier)] - if match: - release_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, match[0]) - release_requires = self.session.get(release_url) - for requires in release_requires.json().get('info', {}).get('requires_dist', {}): - i = InstallRequirement.from_line(requires) + matches = [ + r for r in releases + if '=={0}'.format(r) == str(ireq.req.specifier) + ] + if not matches: + return - if 'extra' not in repr(i.markers): - yield i + release_requires = self.session.get( + 'https://pypi.org/pypi/{0}/{1}/json'.format( + ireq.req.name, matches[0], + ), + ).json() + try: + requires_dist = release_requires['info']['requires_dist'] + except KeyError: + return + + for requires in requires_dist: + i = InstallRequirement.from_line(requires) + if 'extra' not in repr(i.markers): + yield i try: if ireq not in self._json_dep_cache: @@ -239,7 +247,6 @@ class PyPIRepository(BaseRepository): return json_results - def get_legacy_dependencies(self, ireq): """ Given a pinned or an editable InstallRequirement, returns a set of diff --git a/pipenv/utils.py b/pipenv/utils.py index 599d8a0c..ccaa7b6d 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1,20 +1,20 @@ # -*- coding: utf-8 -*- import errno +import logging import os import re -import hashlib -import tempfile -import sys import shutil -import logging +import sys + import crayons import parse import six import stat import warnings -from click import echo as click_echo +from click import echo as click_echo from first import first + try: from weakref import finalize except ImportError: @@ -28,9 +28,10 @@ except ImportError: def detach(self): return False +logging.basicConfig(level=logging.ERROR) + from time import time -logging.basicConfig(level=logging.ERROR) try: from urllib.parse import urlparse except ImportError: @@ -48,7 +49,7 @@ from .pep508checker import lookup from .environments import ( PIPENV_MAX_ROUNDS, PIPENV_CACHE_DIR, - PIPENV_MAX_RETRIES + PIPENV_MAX_RETRIES, ) try: @@ -221,8 +222,6 @@ def actually_resolve_reps( ): from .patched.notpip._internal import basecommand from .patched.notpip._internal.req import parse_requirements - from .patched.notpip._internal.req.req_install import InstallRequirement - from .patched.notpip._vendor import requests as pip_requests from .patched.notpip._internal.exceptions import DistributionNotFound from .patched.notpip._vendor.requests.exceptions import HTTPError from pipenv.patched.piptools.resolver import Resolver @@ -242,18 +241,18 @@ def actually_resolve_reps( req_dir = TemporaryDirectory(suffix='-requirements', prefix='pipenv-') cleanup_req_dir = True for dep in deps: - if dep: - url = None - if ' -i ' in dep: - dep, url = dep.split(' -i ') - req = Requirement.from_line(dep) - _line = req.as_line() - constraints.append(_line) - # extra_constraints = [] - if url: - index_lookup[req.name] = project.get_source(url=url).get('name') - if req.markers: - markers_lookup[req.name] = req.markers.replace('"', "'") + if not dep: + continue + url = None + if ' -i ' in dep: + dep, url = dep.split(' -i ') + req = Requirement.from_line(dep) + constraints.append(req.as_line()) + # extra_constraints = [] + if url: + index_lookup[req.name] = project.get_source(url=url).get('name') + if req.markers: + markers_lookup[req.name] = req.markers.replace('"', "'") constraints_file = None pip_command = get_pip_command() pip_args = [] @@ -274,9 +273,11 @@ def actually_resolve_reps( logging.log.verbose = True piptools_logging.log.verbose = True resolved_tree = set() - piptools_constraints = [c for c in parse_requirements(constraints_file, finder=pypi.finder, session=pypi.session, options=pip_options)] resolver = Resolver( - constraints=piptools_constraints, + constraints=parse_requirements( + constraints_file, + finder=pypi.finder, session=pypi.session, options=pip_options, + ), repository=pypi, clear_caches=clear, prereleases=pre, @@ -1119,7 +1120,7 @@ def extract_uri_from_vcs_dep(dep): return None -def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): +def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): target_dir = os.path.join(src_dir, name) target_rev = vcs_obj.make_rev_options(rev) if not os.path.exists(target_dir): @@ -1128,7 +1129,9 @@ def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): return vcs_obj.get_revision(target_dir) -def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False): +def get_vcs_deps( + project, pip_freeze=None, which=None, verbose=False, clear=False, + pre=False, allow_global=False, dev=False): from .patched.notpip._internal.vcs import VcsSupport section = 'vcs_dev_packages' if dev else 'vcs_packages' lines = [] From 1d54a32dc5d71856470607f5b4feb6c4001af900 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 27 May 2018 02:19:32 -0400 Subject: [PATCH 08/19] Add sources and caches to pip calls Signed-off-by: Dan Ryan --- pipenv/core.py | 9 +++++++-- pipenv/utils.py | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 0f5c7e4f..07a88d66 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1482,7 +1482,7 @@ def pip_install( pre = '--pre' if pre else '' quoted_pip = which_pip(allow_global=allow_global) quoted_pip = escape_grouped_arguments(quoted_pip) - upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else '' + upgrade_strategy = '--upgrade --upgrade-strategy=to-satisfy-only' if selective_upgrade else '' pip_command = '{0} install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format( quoted_pip, install_reqs, @@ -1495,7 +1495,12 @@ def pip_install( ) if verbose: click.echo('$ {0}'.format(pip_command), err=True) - c = delegator.run(pip_command, block=block, env={'PIP_CACHE_DIR': PIPENV_CACHE_DIR}) + pip_config = { + 'PIP_CACHE_DIR': PIPENV_CACHE_DIR, + 'PIP_WHEEL_DIR': os.path.join(PIPENV_CACHE_DIR, 'wheels'), + 'PIP_DESTINATION_DIR': os.path.join(PIPENV_CACHE_DIR, 'pkgs'), + } + c = delegator.run(pip_command, block=block, env=pip_config) return c diff --git a/pipenv/utils.py b/pipenv/utils.py index 599d8a0c..9cb27c78 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -260,6 +260,9 @@ def actually_resolve_reps( if sources: pip_args = prepare_pip_source_args(sources, pip_args) with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f: + if sources: + requirementstxt_sources = ' '.join(pip_args).replace(' --', '\n--') + f.write('{0}\n'.format(requirementstxt_sources)) f.write(u'\n'.join([_constraint for _constraint in constraints])) constraints_file = f.name if verbose: From eae8ac919ce1c96ca73b779e6c8c93d4e1701c7b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 27 May 2018 02:21:39 -0400 Subject: [PATCH 09/19] Update piptools patch Signed-off-by: Dan Ryan --- pipenv/core.py | 7 +- .../vendoring/patches/patched/piptools.patch | 65 ++++++++++--------- 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 07a88d66..f8d03b9e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1505,6 +1505,11 @@ def pip_install( def pip_download(package_name): + pip_config = { + 'PIP_CACHE_DIR': PIPENV_CACHE_DIR, + 'PIP_WHEEL_DIR': os.path.join(PIPENV_CACHE_DIR, 'wheels'), + 'PIP_DESTINATION_DIR': os.path.join(PIPENV_CACHE_DIR, 'pkgs'), + } for source in project.sources: cmd = '{0} download "{1}" -i {2} -d {3}'.format( escape_grouped_arguments(which_pip()), @@ -1512,7 +1517,7 @@ def pip_download(package_name): source['url'], project.download_location, ) - c = delegator.run(cmd, env={'PIP_CACHE_DIR': PIPENV_CACHE_DIR}) + c = delegator.run(cmd, env=pip_config) if c.return_code == 0: break diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index a3220d39..3f6595b4 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..858d697 100644 +index 1c4b943..1ffbf1d 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, @@ -129,20 +129,17 @@ index 1c4b943..858d697 100644 def freshen_build_caches(self): """ -@@ -114,10 +163,21 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +163,18 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match - all_candidates = self.find_all_candidates(ireq.name) -+ _all_candidates = self.find_all_candidates(ireq.name) -+ all_candidates = [] + py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) -+ for c in _all_candidates: -+ if c.requires_python: -+ python_specifier = SpecifierSet(c.requires_python) -+ if not python_specifier.contains(py_version): -+ continue -+ all_candidates.append(c) ++ all_candidates = [ ++ c for c in self.find_all_candidates(ireq.name) ++ if SpecifierSet(c.requires_python).contains(py_version) ++ ] ++ candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) - matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), + try: @@ -153,7 +150,7 @@ index 1c4b943..858d697 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +186,61 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +183,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -174,22 +171,33 @@ index 1c4b943..858d697 100644 + raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq)) + + def gen(ireq): -+ if self.DEFAULT_INDEX_URL in self.finder.index_urls: ++ if self.DEFAULT_INDEX_URL not in self.finder.index_urls: ++ return + -+ url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) -+ r = self.session.get(url) ++ url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) ++ releases = self.session.get(url).json()['releases'] + -+ # TODO: Latest isn't always latest. -+ releases = list(r.json()['releases'].keys()) -+ match = [r for r in releases if '=={0}'.format(r) == str(ireq.req.specifier)] -+ if match: -+ release_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, match[0]) -+ release_requires = self.session.get(release_url) -+ for requires in release_requires.json().get('info', {}).get('requires_dist', {}): -+ i = InstallRequirement.from_line(requires) ++ matches = [ ++ r for r in releases ++ if '=={0}'.format(r) == str(ireq.req.specifier) ++ ] ++ if not matches: ++ return + -+ if 'extra' not in repr(i.markers): -+ yield i ++ release_requires = self.session.get( ++ 'https://pypi.org/pypi/{0}/{1}/json'.format( ++ ireq.req.name, matches[0], ++ ), ++ ).json() ++ try: ++ requires_dist = release_requires['info']['requires_dist'] ++ except KeyError: ++ return ++ ++ for requires in requires_dist: ++ i = InstallRequirement.from_line(requires) ++ if 'extra' not in repr(i.markers): ++ yield i + + try: + if ireq not in self._json_dep_cache: @@ -213,12 +221,11 @@ index 1c4b943..858d697 100644 + + return json_results + -+ + def get_legacy_dependencies(self, ireq): """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -139,6 +249,21 @@ class PyPIRepository(BaseRepository): +@@ -139,6 +256,21 @@ class PyPIRepository(BaseRepository): if not (ireq.editable or is_pinned_requirement(ireq)): raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq)) @@ -240,7 +247,7 @@ index 1c4b943..858d697 100644 if ireq not in self._dependencies_cache: if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # No download_dir for locally available editable requirements. -@@ -164,11 +289,14 @@ class PyPIRepository(BaseRepository): +@@ -164,11 +296,14 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, @@ -257,7 +264,7 @@ index 1c4b943..858d697 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -190,14 +318,44 @@ class PyPIRepository(BaseRepository): +@@ -190,14 +325,44 @@ class PyPIRepository(BaseRepository): upgrade_strategy="to-satisfy-only", force_reinstall=False, ignore_dependencies=False, @@ -304,7 +311,7 @@ index 1c4b943..858d697 100644 reqset.cleanup_files() return set(self._dependencies_cache[ireq]) -@@ -224,17 +382,10 @@ class PyPIRepository(BaseRepository): +@@ -224,17 +389,10 @@ class PyPIRepository(BaseRepository): matching_candidates = candidates_by_version[matching_versions[0]] return { From 9e661519e0ad207c599a89e0b25b970bdde09840 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 27 May 2018 02:27:58 -0400 Subject: [PATCH 10/19] Probably write unicode strings... Signed-off-by: Dan Ryan --- pipenv/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 3741a93f..71cdddff 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -261,7 +261,7 @@ def actually_resolve_reps( with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f: if sources: requirementstxt_sources = ' '.join(pip_args).replace(' --', '\n--') - f.write('{0}\n'.format(requirementstxt_sources)) + f.write(u'{0}\n'.format(requirementstxt_sources)) f.write(u'\n'.join([_constraint for _constraint in constraints])) constraints_file = f.name if verbose: From 8548fbbb3e53f8b7cb3ff492a49e4cf7721e90f9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 27 May 2018 02:35:46 -0400 Subject: [PATCH 11/19] It is important to check for `python_requires` - The specifierset will fail otherwise - We don't want to include only things that have this -- we want to _exclude_ things that do not match it Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 10 ++++---- .../vendoring/patches/patched/piptools.patch | 24 ++++++++++--------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 43b463af..1db380a7 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -164,10 +164,12 @@ class PyPIRepository(BaseRepository): return ireq # return itself as the best match py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) - all_candidates = [ - c for c in self.find_all_candidates(ireq.name) - if SpecifierSet(c.requires_python).contains(py_version) - ] + all_candidates = [] + for c in self.find_all_candidates(ireq.name): + python_specifier = SpecifierSet(c.requires_python) + if not python_specifier.contains(py_version): + continue + all_candidates.append(c) candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) try: diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 3f6595b4..2d86c8be 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..1ffbf1d 100644 +index 1c4b943..a15c23b 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, @@ -129,16 +129,18 @@ index 1c4b943..1ffbf1d 100644 def freshen_build_caches(self): """ -@@ -114,10 +163,18 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +163,20 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match - all_candidates = self.find_all_candidates(ireq.name) + py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) -+ all_candidates = [ -+ c for c in self.find_all_candidates(ireq.name) -+ if SpecifierSet(c.requires_python).contains(py_version) -+ ] ++ all_candidates = [] ++ for c in self.find_all_candidates(ireq.name): ++ python_specifier = SpecifierSet(c.requires_python) ++ if not python_specifier.contains(py_version): ++ continue ++ all_candidates.append(c) + candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) - matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), @@ -150,7 +152,7 @@ index 1c4b943..1ffbf1d 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +183,71 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +185,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -225,7 +227,7 @@ index 1c4b943..1ffbf1d 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -139,6 +256,21 @@ class PyPIRepository(BaseRepository): +@@ -139,6 +258,21 @@ class PyPIRepository(BaseRepository): if not (ireq.editable or is_pinned_requirement(ireq)): raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq)) @@ -247,7 +249,7 @@ index 1c4b943..1ffbf1d 100644 if ireq not in self._dependencies_cache: if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # No download_dir for locally available editable requirements. -@@ -164,11 +296,14 @@ class PyPIRepository(BaseRepository): +@@ -164,11 +298,14 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, @@ -264,7 +266,7 @@ index 1c4b943..1ffbf1d 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -190,14 +325,44 @@ class PyPIRepository(BaseRepository): +@@ -190,14 +327,44 @@ class PyPIRepository(BaseRepository): upgrade_strategy="to-satisfy-only", force_reinstall=False, ignore_dependencies=False, @@ -311,7 +313,7 @@ index 1c4b943..1ffbf1d 100644 reqset.cleanup_files() return set(self._dependencies_cache[ireq]) -@@ -224,17 +389,10 @@ class PyPIRepository(BaseRepository): +@@ -224,17 +391,10 @@ class PyPIRepository(BaseRepository): matching_candidates = candidates_by_version[matching_versions[0]] return { From a4d838902cd0672db70825d47ba2b60982964213 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 27 May 2018 12:56:46 -0400 Subject: [PATCH 12/19] Logic fix Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 1db380a7..cfdc4aa7 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -167,7 +167,7 @@ class PyPIRepository(BaseRepository): all_candidates = [] for c in self.find_all_candidates(ireq.name): python_specifier = SpecifierSet(c.requires_python) - if not python_specifier.contains(py_version): + if c.requires_python and not python_specifier.contains(py_version): continue all_candidates.append(c) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 2d86c8be..1d219032 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -138,7 +138,7 @@ index 1c4b943..a15c23b 100644 + all_candidates = [] + for c in self.find_all_candidates(ireq.name): + python_specifier = SpecifierSet(c.requires_python) -+ if not python_specifier.contains(py_version): ++ if c.requires_python and not python_specifier.contains(py_version): + continue + all_candidates.append(c) + From 6737ab8b03d143b4c76461d013cf0aedb6cc81c9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 27 May 2018 13:00:08 -0400 Subject: [PATCH 13/19] SpecifierSet logic fix Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 3 +-- tasks/vendoring/patches/patched/piptools.patch | 17 ++++++++--------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index cfdc4aa7..e73cb2d0 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -166,8 +166,7 @@ class PyPIRepository(BaseRepository): py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) all_candidates = [] for c in self.find_all_candidates(ireq.name): - python_specifier = SpecifierSet(c.requires_python) - if c.requires_python and not python_specifier.contains(py_version): + if c.requires_python and not SpecifierSet(c.requires_python).contains(py_version): continue all_candidates.append(c) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 1d219032..e3ff1c35 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..a15c23b 100644 +index 1c4b943..29e771e 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, @@ -129,7 +129,7 @@ index 1c4b943..a15c23b 100644 def freshen_build_caches(self): """ -@@ -114,10 +163,20 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +163,19 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -137,8 +137,7 @@ index 1c4b943..a15c23b 100644 + py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3]))) + all_candidates = [] + for c in self.find_all_candidates(ireq.name): -+ python_specifier = SpecifierSet(c.requires_python) -+ if c.requires_python and not python_specifier.contains(py_version): ++ if c.requires_python and not SpecifierSet(c.requires_python).contains(py_version): + continue + all_candidates.append(c) + @@ -152,7 +151,7 @@ index 1c4b943..a15c23b 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +185,71 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +184,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -227,7 +226,7 @@ index 1c4b943..a15c23b 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -139,6 +258,21 @@ class PyPIRepository(BaseRepository): +@@ -139,6 +257,21 @@ class PyPIRepository(BaseRepository): if not (ireq.editable or is_pinned_requirement(ireq)): raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq)) @@ -249,7 +248,7 @@ index 1c4b943..a15c23b 100644 if ireq not in self._dependencies_cache: if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # No download_dir for locally available editable requirements. -@@ -164,11 +298,14 @@ class PyPIRepository(BaseRepository): +@@ -164,11 +297,14 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, @@ -266,7 +265,7 @@ index 1c4b943..a15c23b 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -190,14 +327,44 @@ class PyPIRepository(BaseRepository): +@@ -190,14 +326,44 @@ class PyPIRepository(BaseRepository): upgrade_strategy="to-satisfy-only", force_reinstall=False, ignore_dependencies=False, @@ -313,7 +312,7 @@ index 1c4b943..a15c23b 100644 reqset.cleanup_files() return set(self._dependencies_cache[ireq]) -@@ -224,17 +391,10 @@ class PyPIRepository(BaseRepository): +@@ -224,17 +390,10 @@ class PyPIRepository(BaseRepository): matching_candidates = candidates_by_version[matching_versions[0]] return { From cc68b365e0b7564d17d5717568a6c5e3e5f7c9e1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 27 May 2018 13:35:25 -0400 Subject: [PATCH 14/19] formatting cleanup and posix style paths - fix for windows vcs urls Signed-off-by: Dan Ryan --- pipenv/utils.py | 68 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 71cdddff..e084d6c1 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1133,53 +1133,87 @@ def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): def get_vcs_deps( - project, pip_freeze=None, which=None, verbose=False, clear=False, - pre=False, allow_global=False, dev=False): + project, + pip_freeze=None, + which=None, + verbose=False, + clear=False, + pre=False, + allow_global=False, + dev=False, +): from .patched.notpip._internal.vcs import VcsSupport - section = 'vcs_dev_packages' if dev else 'vcs_packages' + + section = "vcs_dev_packages" if dev else "vcs_packages" lines = [] lockfiles = [] try: packages = getattr(project, section) except AttributeError: return [], [] + src_dir = Path( + os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src")) + ) + src_dir.mkdir(mode=0o775, exist_ok=True) vcs_registry = VcsSupport vcs_uri_map = { - extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')} + extract_uri_from_vcs_dep(v): {"name": k, "ref": v.get("ref")} for k, v in packages.items() } - for line in pip_freeze.strip().split('\n'): + for line in pip_freeze.strip().split("\n"): # if the line doesn't match a vcs dependency in the Pipfile, # ignore it _vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line) if not _vcs_match: continue - pipfile_name = vcs_uri_map[_vcs_match]['name'] - pipfile_rev = vcs_uri_map[_vcs_match]['ref'] - src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src')) - mkdir_p(src_dir) + pipfile_name = vcs_uri_map[_vcs_match]["name"] + pipfile_rev = vcs_uri_map[_vcs_match]["ref"] pipfile_req = Requirement.from_pipfile(pipfile_name, [], packages[pipfile_name]) names = {pipfile_name} backend = vcs_registry()._registry.get(pipfile_req.vcs) # TODO: Why doesn't pip freeze list 'git+git://' formatted urls? - if line.startswith('-e ') and not '{0}+'.format(pipfile_req.vcs) in line: - line = line.replace('-e ', '-e {0}+'.format(pipfile_req.vcs)) + if line.startswith("-e ") and not "{0}+".format(pipfile_req.vcs) in line: + line = line.replace("-e ", "-e {0}+".format(pipfile_req.vcs)) installed = Requirement.from_line(line) __vcs = backend(url=installed.req.uri) names.add(installed.normalized_name) locked_rev = None for _name in names: - locked_rev = install_or_update_vcs(__vcs, src_dir, _name, rev=pipfile_rev) + locked_rev = install_or_update_vcs( + __vcs, src_dir.as_posix(), _name, rev=pipfile_rev + ) if installed.is_vcs: installed.req.ref = locked_rev lockfiles.append({pipfile_name: installed.pipfile_entry[1]}) - pipfile_srcdir = os.path.join(src_dir, pipfile_name) - lockfile_srcdir = os.path.join(src_dir, installed.normalized_name) + pipfile_srcdir = escape_grouped_arguments((src_dir / pipfile_name).as_posix()) + lockfile_srcdir = escape_grouped_arguments( + (src_dir / installed.normalized_name).as_posix() + ) lines.append(line) - if os.path.exists(pipfile_srcdir): - lockfiles.extend(venv_resolve_deps(['-e {0}'.format(pipfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global)) + if pipfile_srcdir.exists(): + lockfiles.extend( + venv_resolve_deps( + ["-e {0}".format(pipfile_srcdir)], + which=which, + verbose=verbose, + project=project, + clear=clear, + pre=pre, + allow_global=allow_global, + ) + ) else: - lockfiles.extend(venv_resolve_deps(['-e {0}'.format(lockfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global)) + lockfiles.extend( + venv_resolve_deps( + ["-e {0}".format(lockfile_srcdir)], + which=which, + verbose=verbose, + project=project, + clear=clear, + pre=pre, + allow_global=allow_global, + ) + ) return lines, lockfiles From e993ee874dfb2579a30de0cdcbdc616a77a287a1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 27 May 2018 13:52:39 -0400 Subject: [PATCH 15/19] Use filesystem compatible encodings for strings Signed-off-by: Dan Ryan --- pipenv/_compat.py | 7 +++++-- pipenv/core.py | 18 +++++++++++------- pipenv/utils.py | 19 ++++++++++++++++--- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 8ccf025a..68d8069d 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -29,6 +29,11 @@ except ImportError: _types.add(type(arg)) return _types.pop() +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + try: from weakref import finalize @@ -51,8 +56,6 @@ if six.PY2: class ResourceWarning(Warning): pass -# -*- coding=utf-8 -*- - def pip_import(module_path, subimport=None, old_path=None): internal = 'pip._internal.{0}'.format(module_path) diff --git a/pipenv/core.py b/pipenv/core.py index f8d03b9e..a14d85bd 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -46,10 +46,12 @@ from .utils import ( rmtree, split_argument, extract_uri_from_vcs_dep, + fs_str, ) from ._compat import ( TemporaryDirectory, - vcs + vcs, + Path ) from .import pep508checker, progress from .environments import ( @@ -1495,20 +1497,22 @@ def pip_install( ) if verbose: click.echo('$ {0}'.format(pip_command), err=True) + cache_dir = Path(PIPENV_CACHE_DIR) pip_config = { - 'PIP_CACHE_DIR': PIPENV_CACHE_DIR, - 'PIP_WHEEL_DIR': os.path.join(PIPENV_CACHE_DIR, 'wheels'), - 'PIP_DESTINATION_DIR': os.path.join(PIPENV_CACHE_DIR, 'pkgs'), + 'PIP_CACHE_DIR': fs_str(cache_dir.as_posix()), + 'PIP_WHEEL_DIR': fs_str(cache_dir.joinpath('wheels').as_posix()), + 'PIP_DESTINATION_DIR': fs_str(cache_dir.joinpath('pkgs').as_posix()), } c = delegator.run(pip_command, block=block, env=pip_config) return c def pip_download(package_name): + cache_dir = Path(PIPENV_CACHE_DIR) pip_config = { - 'PIP_CACHE_DIR': PIPENV_CACHE_DIR, - 'PIP_WHEEL_DIR': os.path.join(PIPENV_CACHE_DIR, 'wheels'), - 'PIP_DESTINATION_DIR': os.path.join(PIPENV_CACHE_DIR, 'pkgs'), + 'PIP_CACHE_DIR': fs_str(cache_dir.as_posix()), + 'PIP_WHEEL_DIR': fs_str(cache_dir.joinpath('wheels').as_posix()), + 'PIP_DESTINATION_DIR': fs_str(cache_dir.joinpath('pkgs').as_posix()), } for source in project.sources: cmd = '{0} download "{1}" -i {2} -d {3}'.format( diff --git a/pipenv/utils.py b/pipenv/utils.py index e084d6c1..051123d6 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1189,10 +1189,9 @@ def get_vcs_deps( lockfiles.append({pipfile_name: installed.pipfile_entry[1]}) pipfile_srcdir = escape_grouped_arguments((src_dir / pipfile_name).as_posix()) lockfile_srcdir = escape_grouped_arguments( - (src_dir / installed.normalized_name).as_posix() - ) + (src_dir / installed.normalized_name)) lines.append(line) - if pipfile_srcdir.exists(): + if os.path.exists(pipfile_srcdir): lockfiles.extend( venv_resolve_deps( ["-e {0}".format(pipfile_srcdir)], @@ -1217,3 +1216,17 @@ def get_vcs_deps( ) ) return lines, lockfiles + + +def fs_str(string): + """Encodes a string into the proper filesystem encoding + + Borrowed from pip-tools + """ + if isinstance(string, str): + return string + assert not isinstance(string, bytes) + return string.encode(_fs_encoding) + + +_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() From 311c90d191850847f8c5164357efdc39e8c9f88d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 27 May 2018 14:26:30 -0400 Subject: [PATCH 16/19] Fix lockfile filename parsing - convert to string properly --- pipenv/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 051123d6..ea4e0136 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1189,7 +1189,8 @@ def get_vcs_deps( lockfiles.append({pipfile_name: installed.pipfile_entry[1]}) pipfile_srcdir = escape_grouped_arguments((src_dir / pipfile_name).as_posix()) lockfile_srcdir = escape_grouped_arguments( - (src_dir / installed.normalized_name)) + (src_dir / installed.normalized_name).as_posix() + ) lines.append(line) if os.path.exists(pipfile_srcdir): lockfiles.extend( From 824f822bc793ff065d636d9db2694a862ce2084e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 27 May 2018 14:34:21 -0400 Subject: [PATCH 17/19] Don't shell escape posix-style paths Signed-off-by: Dan Ryan --- pipenv/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index ea4e0136..a8664ddc 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1187,10 +1187,8 @@ def get_vcs_deps( if installed.is_vcs: installed.req.ref = locked_rev lockfiles.append({pipfile_name: installed.pipfile_entry[1]}) - pipfile_srcdir = escape_grouped_arguments((src_dir / pipfile_name).as_posix()) - lockfile_srcdir = escape_grouped_arguments( - (src_dir / installed.normalized_name).as_posix() - ) + pipfile_srcdir = (src_dir / pipfile_name).as_posix() + lockfile_srcdir = (src_dir / installed.normalized_name).as_posix() lines.append(line) if os.path.exists(pipfile_srcdir): lockfiles.extend( From a5e69e4a61ba24c702872288fa98a41ba5f29234 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 28 May 2018 13:21:22 +0900 Subject: [PATCH 18/19] Comments --- pipenv/patched/piptools/repositories/pypi.py | 3 +++ pipenv/utils.py | 5 +++++ tasks/vendoring/__init__.py | 5 ++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index e73cb2d0..c435be07 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -266,6 +266,9 @@ class PyPIRepository(BaseRepository): setup_requires = self.finder.get_extras_links( dist.get_metadata_lines('requires.txt') ) + # HACK: Sometimes the InstallRequirement doesn't properly get + # these values set on it during the resolution process. It's + # difficult to pin down what is going wrong. This fixes things. ireq.version = dist.version ireq.project_name = dist.project_name ireq.req = dist.as_requirement() diff --git a/pipenv/utils.py b/pipenv/utils.py index a8664ddc..27ce9d30 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -247,8 +247,13 @@ def actually_resolve_reps( if ' -i ' in dep: dep, url = dep.split(' -i ') req = Requirement.from_line(dep) + + # req.as_line() is theoratically the same as dep, but is guarenteed to + # be normalized. This is safer than passing in dep. + # TODO: Stop passing dep lines around; just use requirement objects. constraints.append(req.as_line()) # extra_constraints = [] + if url: index_lookup[req.name] = project.get_source(url=url).get('name') if req.markers: diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 9bd14dee..4b64b818 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -468,7 +468,10 @@ def generate_patch(ctx, package_path, patch_description, base='HEAD'): pkg = Path(package_path) if len(pkg.parts) != 2 or pkg.parts[0] not in ('vendor', 'patched'): raise ValueError('example usage: generate-patch patched/pew some-description') - patch_fn = '{0}-{1}.patch'.format(pkg.parts[1], patch_description) + if patch_description: + patch_fn = '{0}-{1}.patch'.format(pkg.parts[1], patch_description) + else: + patch_fn = '{0}.patch'.format(pkg.parts[1]) command = 'git diff {base} -p {root} > {out}'.format( base=base, root=Path('pipenv').joinpath(pkg), From b60548172b2a6f9199cdde703e917fc801591877 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 28 May 2018 00:55:17 -0400 Subject: [PATCH 19/19] Update patch Signed-off-by: Dan Ryan --- tasks/vendoring/patches/patched/piptools.patch | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index e3ff1c35..fe2c46d8 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..29e771e 100644 +index 1c4b943..aae0122 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, @@ -226,7 +226,7 @@ index 1c4b943..29e771e 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -139,6 +257,21 @@ class PyPIRepository(BaseRepository): +@@ -139,6 +257,24 @@ class PyPIRepository(BaseRepository): if not (ireq.editable or is_pinned_requirement(ireq)): raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq)) @@ -239,6 +239,9 @@ index 1c4b943..29e771e 100644 + setup_requires = self.finder.get_extras_links( + dist.get_metadata_lines('requires.txt') + ) ++ # HACK: Sometimes the InstallRequirement doesn't properly get ++ # these values set on it during the resolution process. It's ++ # difficult to pin down what is going wrong. This fixes things. + ireq.version = dist.version + ireq.project_name = dist.project_name + ireq.req = dist.as_requirement() @@ -248,7 +251,7 @@ index 1c4b943..29e771e 100644 if ireq not in self._dependencies_cache: if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # No download_dir for locally available editable requirements. -@@ -164,11 +297,14 @@ class PyPIRepository(BaseRepository): +@@ -164,11 +300,14 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, @@ -265,7 +268,7 @@ index 1c4b943..29e771e 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -190,14 +326,44 @@ class PyPIRepository(BaseRepository): +@@ -190,14 +329,44 @@ class PyPIRepository(BaseRepository): upgrade_strategy="to-satisfy-only", force_reinstall=False, ignore_dependencies=False, @@ -312,7 +315,7 @@ index 1c4b943..29e771e 100644 reqset.cleanup_files() return set(self._dependencies_cache[ireq]) -@@ -224,17 +390,10 @@ class PyPIRepository(BaseRepository): +@@ -224,17 +393,10 @@ class PyPIRepository(BaseRepository): matching_candidates = candidates_by_version[matching_versions[0]] return {