Merge pull request #2212 from pypa/update-piptools

Update piptools (work in progress)
This commit is contained in:
Dan Ryan
2018-05-23 17:51:30 -04:00
committed by GitHub
27 changed files with 406 additions and 325 deletions
+1 -1
View File
@@ -3,6 +3,6 @@ git+https://github.com/jumpscale7/python-consistent-toml.git#egg=contoml
crayons==0.1.2
git+https://github.com/berdario/pew.git@1.1.5#egg=pew
pipfile==0.0.2
pip-tools==2.0.1
git+https://github.com/jazzband/pip-tools.git@9cb41d828fcb0967a32cc140c1dcaca94e5f4daa#egg=piptools
prettytoml==0.3
pip==9.0.3
+17
View File
@@ -0,0 +1,17 @@
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-6
View File
@@ -1,6 +0,0 @@
import os
import sys
# Inject vendored directory into system path.
v_path = os.path.abspath(os.path.sep.join([os.path.dirname(os.path.realpath(__file__)), '_vendored']))
sys.path.insert(0, v_path)
@@ -11,3 +11,21 @@ if six.PY2:
else:
from tempfile import TemporaryDirectory
from contextlib import ExitStack
from .pip_compat import (
InstallRequirement,
parse_requirements,
RequirementSet,
user_cache_dir,
FAVORITE_HASH,
is_file_url,
url_to_path,
PackageFinder,
FormatControl,
Wheel,
Command,
cmdoptions,
get_installed_distributions,
PyPI,
SafeFileCache,
)
@@ -0,0 +1,40 @@
# -*- coding=utf-8 -*-
import importlib
def do_import(module_path, subimport=None, old_path=None, vendored_name=None):
internal = 'pip._internal.{0}'.format(module_path)
old_path = old_path or module_path
pip9 = 'pip.{0}'.format(old_path)
_tmp = None
if vendored_name:
vendor = '{0}.{1}'.format(vendored_name, old_path)
try:
_tmp = importlib.import_module(vendor)
except ImportError:
pass
if not _tmp:
try:
_tmp = importlib.import_module(internal)
except ImportError:
_tmp = importlib.import_module(pip9)
if subimport:
return getattr(_tmp, subimport, _tmp)
return _tmp
InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name='notpip')
parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name='notpip')
RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name='notpip')
user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name='notpip')
FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name='notpip')
is_file_url = do_import('download', 'is_file_url', vendored_name='notpip')
url_to_path = do_import('download', 'url_to_path', vendored_name='notpip')
PackageFinder = do_import('index', 'PackageFinder', vendored_name='notpip')
FormatControl = do_import('index', 'FormatControl', vendored_name='notpip')
Wheel = do_import('wheel', 'Wheel', vendored_name='notpip')
Command = do_import('basecommand', 'Command', vendored_name='pip9')
cmdoptions = do_import('cmdoptions', vendored_name='pip9')
get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name='pip9')
PyPI = do_import('models.index', 'PyPI', vendored_name='notpip')
SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip')
+1 -1
View File
@@ -6,7 +6,7 @@ import json
import os
import sys
from pip9._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.requirements import Requirement
from .exceptions import PipToolsError
from .locations import CACHE_DIR
+28 -6
View File
@@ -3,22 +3,44 @@ class PipToolsError(Exception):
class NoCandidateFound(PipToolsError):
def __init__(self, ireq, candidates_tried, index_urls):
def __init__(self, ireq, candidates_tried, finder):
self.ireq = ireq
self.candidates_tried = candidates_tried
self.index_urls = index_urls
self.finder = finder
def __str__(self):
sorted_versions = sorted(c.version for c in self.candidates_tried)
versions = []
pre_versions = []
for candidate in sorted(self.candidates_tried):
version = str(candidate.version)
if candidate.version.is_prerelease:
pre_versions.append(version)
else:
versions.append(version)
lines = [
'Could not find a version that matches {}'.format(self.ireq),
'Tried: {}'.format(', '.join(str(version) for version in sorted_versions) or '(no version found at all)')
]
if sorted_versions:
if versions:
lines.append('Tried: {}'.format(', '.join(versions)))
if pre_versions:
if self.finder.allow_all_prereleases:
line = 'Tried'
else:
line = 'Skipped'
line += ' pre-versions: {}'.format(', '.join(pre_versions))
lines.append(line)
if versions or pre_versions:
lines.append('There are incompatible versions in the resolved dependencies.')
else:
lines.append('No versions found')
lines.append('{} {} reachable?'.format(
'Were' if len(self.index_urls) > 1 else 'Was', ' or '.join(self.index_urls))
'Were' if len(self.finder.index_urls) > 1 else 'Was', ' or '.join(self.finder.index_urls))
)
return '\n'.join(lines)
+1 -1
View File
@@ -3,7 +3,7 @@ from shutil import rmtree
from .click import secho
# Patch by vphilippon 2017-11-22: Use pipenv cache path.
# from pip9.utils.appdirs import user_cache_dir
# from ._compat import user_cache_dir
from pipenv.environments import PIPENV_CACHE_DIR
# The user_cache_dir helper comes straight from pip itself
+1 -1
View File
@@ -44,5 +44,5 @@ class BaseRepository(object):
@contextmanager
def allow_all_wheels(self):
"""
Monkey patches pip9.Wheel to allow wheels from all platforms and Python versions.
Monkey patches pip.Wheel to allow wheels from all platforms and Python versions.
"""
@@ -6,7 +6,7 @@ from contextlib import contextmanager
from piptools.utils import as_tuple, key_from_req, make_install_requirement
from .base import BaseRepository
from pip9.utils.hashes import FAVORITE_HASH
from .._compat import FAVORITE_HASH
def ireq_satisfied_by_existing_pin(ireq, existing_pin):
+86 -33
View File
@@ -7,17 +7,22 @@ import os
from contextlib import contextmanager
from shutil import rmtree
from notpip.download import is_file_url, url_to_path
from notpip.index import PackageFinder
from notpip.req.req_set import RequirementSet
from notpip.wheel import Wheel
from notpip.req.req_install import InstallRequirement
from .._compat import (
is_file_url,
url_to_path,
PackageFinder,
RequirementSet,
Wheel,
FAVORITE_HASH,
TemporaryDirectory,
PyPI,
InstallRequirement,
SafeFileCache,
)
from pip9._vendor.packaging.requirements import InvalidRequirement
from pip9._vendor.pyparsing import ParseException
from notpip.download import SafeFileCache
from notpip.utils.hashes import FAVORITE_HASH
from .._compat import TemporaryDirectory
from ..cache import CACHE_DIR
from pipenv.environments import PIPENV_CACHE_DIR
from ..exceptions import NoCandidateFound
@@ -26,11 +31,23 @@ from ..utils import (fs_str, is_pinned_requirement, lookup_table,
from .base import BaseRepository
try:
from pip._internal.operations.prepare import RequirementPreparer
from pip._internal.resolve import Resolver as PipResolver
except ImportError:
pass
try:
from pip._internal.cache import WheelCache
except ImportError:
from pip.wheel import WheelCache
class HashCache(SafeFileCache):
"""Caches hashes of PyPI artifacts so we do not need to re-download them
Hashes are only cached when the URL appears to contain a hash in it (and the cache key includes
the hash value returned from the server). This ought to avoid issues where the location on the
Hashes are only cached when the URL appears to contain a hash in it and the cache key includes
the hash value returned from the server). This ought to avoid ssues where the location on the
server changes."""
def __init__(self, *args, **kwargs):
session = kwargs.pop('session')
@@ -39,7 +56,7 @@ class HashCache(SafeFileCache):
super(HashCache, self).__init__(*args, **kwargs)
def get_hash(self, location):
# if there is no location hash (i.e., md5 / sha256 / etc) we don't want to store it
# if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it
hash_value = None
can_hash = location.hash
if can_hash:
@@ -61,7 +78,7 @@ class HashCache(SafeFileCache):
class PyPIRepository(BaseRepository):
DEFAULT_INDEX_URL = 'https://pypi.org/simple'
DEFAULT_INDEX_URL = PyPI.simple_url
"""
The PyPIRepository will use the provided Finder instance to lookup
@@ -72,6 +89,8 @@ class PyPIRepository(BaseRepository):
def __init__(self, pip_options, session, use_json=False):
self.session = session
self.use_json = use_json
self.pip_options = pip_options
self.wheel_cache = WheelCache(PIPENV_CACHE_DIR, pip_options.format_control)
index_urls = [pip_options.index_url] + pip_options.extra_index_urls
if pip_options.no_index:
@@ -103,8 +122,8 @@ class PyPIRepository(BaseRepository):
# Setup file paths
self.freshen_build_caches()
self._download_dir = fs_str(os.path.join(CACHE_DIR, 'pkgs'))
self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, 'wheels'))
self._download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'pkgs'))
self._wheel_download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'wheels'))
def freshen_build_caches(self):
"""
@@ -148,7 +167,7 @@ class PyPIRepository(BaseRepository):
# Reuses pip's internal candidate sort key to sort
matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
if not matching_candidates:
raise NoCandidateFound(ireq, all_candidates, self.finder.index_urls)
raise NoCandidateFound(ireq, all_candidates, self.finder)
best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)
# Turn the candidate into a pinned InstallRequirement
@@ -174,8 +193,9 @@ class PyPIRepository(BaseRepository):
# TODO: Latest isn't always latest.
latest = list(r.json()['releases'].keys())[-1]
if str(ireq.req.specifier) == '=={0}'.format(latest):
for requires in r.json().get('info', {}).get('requires_dist', {}):
latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest)
latest_requires = self.session.get(latest_url)
for requires in latest_requires.json().get('info', {}).get('requires_dist', {}):
i = InstallRequirement.from_line(requires)
if 'extra' not in repr(i.markers):
@@ -189,7 +209,6 @@ class PyPIRepository(BaseRepository):
except Exception:
return set()
def get_dependencies(self, ireq):
json_results = set()
@@ -226,7 +245,6 @@ class PyPIRepository(BaseRepository):
except TypeError:
pass
if ireq not in self._dependencies_cache:
if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)):
# No download_dir for locally available editable requirements.
@@ -244,17 +262,52 @@ class PyPIRepository(BaseRepository):
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
reqset = RequirementSet(self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_compatibility=False
)
result = reqset._prepare_file(self.finder, ireq, ignore_requires_python=True)
try:
# Pip < 9 and below
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_compatibility=False,
wheel_cache=self.wheel_cache,
)
result = reqset._prepare_file(
self.finder,
ireq,
ignore_requires_python=True
)
except TypeError:
# Pip >= 10 (new resolver!)
preparer = RequirementPreparer(
build_dir=self.build_dir,
src_dir=self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
progress_bar='off',
build_isolation=False
)
reqset = RequirementSet()
ireq.is_direct = True
reqset.add_requirement(ireq)
self.resolver = PipResolver(
preparer=preparer,
finder=self.finder,
session=self.session,
upgrade_strategy="to-satisfy-only",
force_reinstall=False,
ignore_dependencies=False,
ignore_requires_python=False,
ignore_installed=True,
isolated=False,
wheel_cache=self.wheel_cache,
use_user_site=False,
ignore_compatibility=False
)
self.resolver.resolve(reqset)
result = reqset.requirements.values()
# Convert setup_requires dict into a somewhat usable form.
if setup_requires:
for section in setup_requires:
@@ -279,12 +332,12 @@ class PyPIRepository(BaseRepository):
pass
if reqset.requires_python:
marker = 'python_version=="{0}"'.format(reqset.requires_python.replace(' ', ''))
new_req = InstallRequirement.from_line('{0}; {1}'.format(str(ireq.req), marker))
result = [new_req]
self._dependencies_cache[ireq] = result
reqset.cleanup_files()
return set(self._dependencies_cache[ireq])
def get_hashes(self, ireq):
@@ -317,7 +370,7 @@ class PyPIRepository(BaseRepository):
@contextmanager
def allow_all_wheels(self):
"""
Monkey patches pip9.Wheel to allow wheels from all platforms and Python versions.
Monkey patches pip.Wheel to allow wheels from all platforms and Python versions.
This also saves the candidate cache and set a new one, or else the results from the
previous non-patched calls will interfere.
@@ -351,7 +404,7 @@ def open_local_or_remote_file(link, session):
"""
Open local or remote file for reading.
:type link: pip9.index.Link
:type link: pip.index.Link
:type session: requests.Session
:raises ValueError: If link points to a local directory.
:return: a context manager to the opened file-like object
+2 -3
View File
@@ -8,7 +8,7 @@ from itertools import chain, count
import os
from first import first
from pip9.req import InstallRequirement
from ._compat import InstallRequirement
from . import click
from .cache import DependencyCache
@@ -302,7 +302,7 @@ class Resolver(object):
dependency_strings = self.dependency_cache[ireq]
log.debug(' {:25} requires {}'.format(format_requirement(ireq),
', '.join(sorted(dependency_strings, key=lambda s: s.lower())) or '-'))
from notpip._vendor.packaging.markers import InvalidMarker
from pip9._vendor.packaging.markers import InvalidMarker
for dependency_string in dependency_strings:
try:
_dependency_string = dependency_string
@@ -315,7 +315,6 @@ class Resolver(object):
except InvalidMarker:
yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint)
def reverse_dependencies(self, ireqs):
non_editable = [ireq for ireq in ireqs if not ireq.editable]
return self.dependency_cache.reverse_dependencies(non_editable)
+13 -7
View File
@@ -7,8 +7,12 @@ import os
import sys
import tempfile
import pip9
from pip9.req import InstallRequirement, parse_requirements
from .._compat import (
InstallRequirement,
parse_requirements,
cmdoptions,
Command,
)
from .. import click
from ..exceptions import PipToolsError
@@ -21,7 +25,7 @@ from ..writer import OutputWriter
DEFAULT_REQUIREMENTS_FILE = 'requirements.in'
class PipCommand(pip9.basecommand.Command):
class PipCommand(Command):
name = 'PipCommand'
@@ -247,12 +251,14 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url,
def get_pip_command():
# Use pip's parser for pip9.conf management and defaults.
# Use pip's parser for pip.conf management and defaults.
# General options (find_links, index_url, extra_index_url, trusted_host,
# and pre) are defered to pip9.
# and pre) are defered to pip.
pip_command = PipCommand()
index_opts = pip9.cmdoptions.make_option_group(
pip9.cmdoptions.index_group,
pip_command.parser.add_option(cmdoptions.no_binary())
pip_command.parser.add_option(cmdoptions.only_binary())
index_opts = cmdoptions.make_option_group(
cmdoptions.index_group,
pip_command.parser,
)
pip_command.parser.insert_option_group(0, index_opts)
+3 -3
View File
@@ -5,9 +5,9 @@ from __future__ import (absolute_import, division, print_function,
import os
import sys
import pip9
from .. import click, sync
from .._compat import parse_requirements, get_installed_distributions
from ..exceptions import PipToolsError
from ..logging import log
from ..utils import flat_map
@@ -45,7 +45,7 @@ def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet,
log.error('ERROR: ' + msg)
sys.exit(2)
requirements = flat_map(lambda src: pip9.req.parse_requirements(src, session=True),
requirements = flat_map(lambda src: parse_requirements(src, session=True),
src_files)
try:
@@ -54,7 +54,7 @@ def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet,
log.error(str(e))
sys.exit(2)
installed_dists = pip9.get_installed_distributions(skip=[], user_only=user_only)
installed_dists = get_installed_distributions(skip=[], user_only=user_only)
to_install, to_uninstall = sync.diff(requirements, installed_dists)
install_flags = []
+17 -1
View File
@@ -2,11 +2,13 @@
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import os
import sys
from itertools import chain, groupby
from collections import OrderedDict
from contextlib import contextmanager
from pip9.req import InstallRequirement
from ._compat import InstallRequirement
from first import first
@@ -270,3 +272,17 @@ def fs_str(string):
_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
# Borrowed from pew to avoid importing pew which imports psutil
# See https://github.com/berdario/pew/blob/master/pew/_utils.py#L82
@contextmanager
def temp_environ():
"""Allow the ability to set os.environ temporarily"""
environ = dict(os.environ)
try:
yield
finally:
os.environ.clear()
os.environ.update(environ)
+1 -1
View File
@@ -15,7 +15,7 @@ try:
from pip import get_installed_distributions
except ImportError:
# pip 10
from pip9._internal.utils.misc import get_installed_distributions
from pip._internal.utils.misc import get_installed_distributions
@click.group()
-5
View File
@@ -309,11 +309,6 @@ def vendor(ctx, vendor_dir, rewrite=True):
log('Renaming specified libs...')
for item in vendor_dir.iterdir():
if item.is_dir():
if item.name == 'requests' and not (item / 'cacert.pem').exists():
if 'certifi' in vendored_libs:
cert = vendor_dir / 'certifi' / 'cacert.pem'
copy_to = item / 'cacert.pem'
copy_to.write_bytes(cert.read_bytes())
if rewrite:
log('Rewriting imports for %s...' % item)
rewrite_imports(item, vendored_libs, vendor_dir)
@@ -174,95 +174,3 @@ index 4254fbd..a7efaa0 100644
DecodeError, ReadTimeoutError, ProtocolError, LocationParseError)
from io import UnsupportedOperation
diff --git a/pipenv/patched/piptools/cache.py b/pipenv/patched/piptools/cache.py
index 7595b96..5a44953 100644
--- a/pipenv/patched/piptools/cache.py
+++ b/pipenv/patched/piptools/cache.py
@@ -6,7 +6,7 @@ import json
import os
import sys
-from pip._vendor.packaging.requirements import Requirement
+from pip9._vendor.packaging.requirements import Requirement
from .exceptions import PipToolsError
from .locations import CACHE_DIR
diff --git a/pipenv/patched/piptools/repositories/base.py b/pipenv/patched/piptools/repositories/base.py
index 57e85fd..69835c0 100644
--- a/pipenv/patched/piptools/repositories/base.py
+++ b/pipenv/patched/piptools/repositories/base.py
@@ -44,5 +44,5 @@ class BaseRepository(object):
@contextmanager
def allow_all_wheels(self):
"""
- Monkey patches pip.Wheel to allow wheels from all platforms and Python versions.
+ Monkey patches pip9.Wheel to allow wheels from all platforms and Python versions.
"""
diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py
index 8f6f028..3c6182b 100644
--- a/pipenv/patched/piptools/repositories/local.py
+++ b/pipenv/patched/piptools/repositories/local.py
@@ -6,7 +6,7 @@ from contextlib import contextmanager
from piptools.utils import as_tuple, key_from_req, make_install_requirement
from .base import BaseRepository
-from pip.utils.hashes import FAVORITE_HASH
+from pip9.utils.hashes import FAVORITE_HASH
def ireq_satisfied_by_existing_pin(ireq, existing_pin):
diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
index e1f63d2..03833a4 100644
--- a/pipenv/patched/piptools/repositories/pypi.py
+++ b/pipenv/patched/piptools/repositories/pypi.py
@@ -317,7 +317,7 @@ class PyPIRepository(BaseRepository):
@contextmanager
def allow_all_wheels(self):
"""
- Monkey patches pip.Wheel to allow wheels from all platforms and Python versions.
+ Monkey patches pip9.Wheel to allow wheels from all platforms and Python versions.
This also saves the candidate cache and set a new one, or else the results from the
previous non-patched calls will interfere.
diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py
index 862be14..64b1199 100644
--- a/pipenv/patched/piptools/resolver.py
+++ b/pipenv/patched/piptools/resolver.py
@@ -8,7 +8,7 @@ from itertools import chain, count
import os
from first import first
-from pip.req import InstallRequirement
+from pip9.req import InstallRequirement
from . import click
from .cache import DependencyCache
diff --git a/pipenv/patched/piptools/scripts/compile.py b/pipenv/patched/piptools/scripts/compile.py
index 0e83bfc..8194fcf 100644
--- a/pipenv/patched/piptools/scripts/compile.py
+++ b/pipenv/patched/piptools/scripts/compile.py
@@ -247,9 +247,9 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url,
def get_pip_command():
- # Use pip's parser for pip.conf management and defaults.
+ # Use pip's parser for pip9.conf management and defaults.
# General options (find_links, index_url, extra_index_url, trusted_host,
- # and pre) are defered to pip.
+ # and pre) are defered to pip9.
pip_command = PipCommand()
index_opts = pip9.cmdoptions.make_option_group(
pip9.cmdoptions.index_group,
diff --git a/pipenv/patched/safety/cli.py b/pipenv/patched/safety/cli.py
index 37ae3ac..fc543ef 100644
--- a/pipenv/patched/safety/cli.py
+++ b/pipenv/patched/safety/cli.py
@@ -15,7 +15,7 @@ try:
from pip import get_installed_distributions
except ImportError:
# pip 10
- from pip._internal.utils.misc import get_installed_distributions
+ from pip9._internal.utils.misc import get_installed_distributions
@click.group()
+148 -146
View File
@@ -1,14 +1,14 @@
diff --git a/pipenv/patched/piptools/locations.py b/pipenv/patched/piptools/locations.py
index aa0610b..5791f0f 100644
index 4e6174c..75f9b49 100644
--- a/pipenv/patched/piptools/locations.py
+++ b/pipenv/patched/piptools/locations.py
@@ -2,10 +2,13 @@ import os
from shutil import rmtree
from .click import secho
-from pip.utils.appdirs import user_cache_dir
-from ._compat import user_cache_dir
+# Patch by vphilippon 2017-11-22: Use pipenv cache path.
+# from pip9.utils.appdirs import user_cache_dir
+# from ._compat import user_cache_dir
+from pipenv.environments import PIPENV_CACHE_DIR
# The user_cache_dir helper comes straight from pip itself
@@ -18,43 +18,37 @@ index aa0610b..5791f0f 100644
# NOTE
# We used to store the cache dir under ~/.pip-tools, which is not the
diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
index d3b7fe7..e1f63d2 100644
--- a/pipenv/patched/piptools/repositories/pypi.py
+++ b/pipenv/patched/piptools/repositories/pypi.py
@@ -7,20 +7,59 @@ import os
from contextlib import contextmanager
from shutil import rmtree
diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py
index 1c4b943..1c808f3 100644
--- a/piptools/repositories/pypi.py
+++ b/piptools/repositories/pypi.py
@@ -15,10 +15,16 @@ from .._compat import (
Wheel,
FAVORITE_HASH,
TemporaryDirectory,
- PyPI
+ PyPI,
+ InstallRequirement,
+ SafeFileCache,
)
-from pip.download import is_file_url, url_to_path
-from pip.index import PackageFinder
-from pip.req.req_set import RequirementSet
-from pip.wheel import Wheel
-from pip.utils.hashes import FAVORITE_HASH
+from notpip.download import is_file_url, url_to_path
+from notpip.index import PackageFinder
+from notpip.req.req_set import RequirementSet
+from notpip.wheel import Wheel
+from notpip.req.req_install import InstallRequirement
+from pip9._vendor.packaging.requirements import InvalidRequirement
+from pip9._vendor.pyparsing import ParseException
+from notpip.download import SafeFileCache
+from notpip.utils.hashes import FAVORITE_HASH
from .._compat import TemporaryDirectory
+
from ..cache import CACHE_DIR
+from pipenv.environments import PIPENV_CACHE_DIR
from ..exceptions import NoCandidateFound
from ..utils import (fs_str, is_pinned_requirement, lookup_table,
make_install_requirement)
from .base import BaseRepository
@@ -37,6 +43,40 @@ except ImportError:
from pip.wheel import WheelCache
+class HashCache(SafeFileCache):
+ """Caches hashes of PyPI artifacts so we do not need to re-download them
+
+ Hashes are only cached when the URL appears to contain a hash in it (and the cache key includes
+ the hash value returned from the server). This ought to avoid issues where the location on the
+ Hashes are only cached when the URL appears to contain a hash in it and the cache key includes
+ the hash value returned from the server). This ought to avoid ssues where the location on the
+ server changes."""
+ def __init__(self, *args, **kwargs):
+ session = kwargs.pop('session')
@@ -63,7 +57,7 @@ index d3b7fe7..e1f63d2 100644
+ super(HashCache, self).__init__(*args, **kwargs)
+
+ def get_hash(self, location):
+ # if there is no location hash (i.e., md5 / sha256 / etc) we don't want to store it
+ # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it
+ hash_value = None
+ can_hash = location.hash
+ if can_hash:
@@ -85,10 +79,9 @@ index d3b7fe7..e1f63d2 100644
+
+
class PyPIRepository(BaseRepository):
- DEFAULT_INDEX_URL = 'https://pypi.python.org/simple'
+ DEFAULT_INDEX_URL = 'https://pypi.org/simple'
DEFAULT_INDEX_URL = PyPI.simple_url
@@ -30,8 +69,9 @@ class PyPIRepository(BaseRepository):
@@ -46,10 +86,11 @@ class PyPIRepository(BaseRepository):
config), but any other PyPI mirror can be used if index_urls is
changed/configured on the Finder.
"""
@@ -96,10 +89,13 @@ index d3b7fe7..e1f63d2 100644
+ def __init__(self, pip_options, session, use_json=False):
self.session = session
+ self.use_json = use_json
self.pip_options = pip_options
- self.wheel_cache = WheelCache(CACHE_DIR, pip_options.format_control)
+ self.wheel_cache = WheelCache(PIPENV_CACHE_DIR, pip_options.format_control)
index_urls = [pip_options.index_url] + pip_options.extra_index_urls
if pip_options.no_index:
@@ -56,6 +96,10 @@ class PyPIRepository(BaseRepository):
@@ -74,11 +115,15 @@ class PyPIRepository(BaseRepository):
# of all secondary dependencies for the given requirement, so we
# only have to go to disk once for each requirement
self._dependencies_cache = {}
@@ -110,7 +106,14 @@ index d3b7fe7..e1f63d2 100644
# Setup file paths
self.freshen_build_caches()
@@ -108,11 +152,60 @@ class PyPIRepository(BaseRepository):
- self._download_dir = fs_str(os.path.join(CACHE_DIR, 'pkgs'))
- self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, 'wheels'))
+ self._download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'pkgs'))
+ self._wheel_download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'wheels'))
def freshen_build_caches(self):
"""
@@ -126,11 +171,60 @@ class PyPIRepository(BaseRepository):
best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)
# Turn the candidate into a pinned InstallRequirement
@@ -139,8 +142,9 @@ index d3b7fe7..e1f63d2 100644
+ # TODO: Latest isn't always latest.
+ latest = list(r.json()['releases'].keys())[-1]
+ if str(ireq.req.specifier) == '=={0}'.format(latest):
+
+ for requires in r.json().get('info', {}).get('requires_dist', {}):
+ latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest)
+ latest_requires = self.session.get(latest_url)
+ for requires in latest_requires.json().get('info', {}).get('requires_dist', {}):
+ i = InstallRequirement.from_line(requires)
+
+ if 'extra' not in repr(i.markers):
@@ -153,7 +157,6 @@ index d3b7fe7..e1f63d2 100644
+ return set(self._json_dep_cache[ireq])
+ except Exception:
+ return set()
+
def get_dependencies(self, ireq):
+ json_results = set()
@@ -174,7 +177,7 @@ index d3b7fe7..e1f63d2 100644
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
@@ -121,6 +214,19 @@ class PyPIRepository(BaseRepository):
@@ -139,6 +233,18 @@ class PyPIRepository(BaseRepository):
if not (ireq.editable or is_pinned_requirement(ireq)):
raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq))
@@ -189,24 +192,36 @@ index d3b7fe7..e1f63d2 100644
+ )
+ except TypeError:
+ pass
+
+
if ireq not in self._dependencies_cache:
if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)):
# No download_dir for locally available editable requirements.
@@ -142,8 +248,43 @@ class PyPIRepository(BaseRepository):
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
- session=self.session)
- self._dependencies_cache[ireq] = reqset._prepare_file(self.finder, ireq)
+ session=self.session,
+ ignore_installed=True,
+ ignore_compatibility=False
+ )
+
+ result = reqset._prepare_file(self.finder, ireq, ignore_requires_python=True)
+
@@ -164,11 +270,14 @@ class PyPIRepository(BaseRepository):
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
+ ignore_installed=True,
+ ignore_compatibility=False,
wheel_cache=self.wheel_cache,
)
- self._dependencies_cache[ireq] = reqset._prepare_file(
+ result = reqset._prepare_file(
self.finder,
- ireq
+ ireq,
+ ignore_requires_python=True
)
except TypeError:
# Pip >= 10 (new resolver!)
@@ -195,9 +304,39 @@ class PyPIRepository(BaseRepository):
isolated=False,
wheel_cache=self.wheel_cache,
use_user_site=False,
+ ignore_compatibility=False
)
self.resolver.resolve(reqset)
- self._dependencies_cache[ireq] = reqset.requirements.values()
+ result = reqset.requirements.values()
+ # Convert setup_requires dict into a somewhat usable form.
+ if setup_requires:
+ for section in setup_requires:
@@ -231,16 +246,15 @@ index d3b7fe7..e1f63d2 100644
+ pass
+
+ if reqset.requires_python:
+
+ marker = 'python_version=="{0}"'.format(reqset.requires_python.replace(' ', ''))
+ new_req = InstallRequirement.from_line('{0}; {1}'.format(str(ireq.req), marker))
+ result = [new_req]
+
+ self._dependencies_cache[ireq] = result
reqset.cleanup_files()
return set(self._dependencies_cache[ireq])
def get_hashes(self, ireq):
@@ -169,17 +310,10 @@ class PyPIRepository(BaseRepository):
@@ -224,17 +363,10 @@ class PyPIRepository(BaseRepository):
matching_candidates = candidates_by_version[matching_versions[0]]
return {
@@ -259,17 +273,8 @@ index d3b7fe7..e1f63d2 100644
@contextmanager
def allow_all_wheels(self):
"""
@@ -217,7 +351,7 @@ def open_local_or_remote_file(link, session):
"""
Open local or remote file for reading.
- :type link: pip.index.Link
+ :type link: pip9.index.Link
:type session: requests.Session
:raises ValueError: If link points to a local directory.
:return: a context manager to the opened file-like object
diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py
index 8c4e981..862be14 100644
index 05ec8fd..6380e8f 100644
--- a/pipenv/patched/piptools/resolver.py
+++ b/pipenv/patched/piptools/resolver.py
@@ -14,7 +14,7 @@ from . import click
@@ -332,7 +337,7 @@ index 8c4e981..862be14 100644
elif not is_pinned_requirement(ireq):
raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq))
@@ -283,14 +295,26 @@ class Resolver(object):
@@ -283,14 +295,25 @@ class Resolver(object):
if ireq not in self.dependency_cache:
log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow')
dependencies = self.repository.get_dependencies(ireq)
@@ -344,7 +349,7 @@ index 8c4e981..862be14 100644
dependency_strings = self.dependency_cache[ireq]
log.debug(' {:25} requires {}'.format(format_requirement(ireq),
', '.join(sorted(dependency_strings, key=lambda s: s.lower())) or '-'))
+ from notpip._vendor.packaging.markers import InvalidMarker
+ from pip9._vendor.packaging.markers import InvalidMarker
for dependency_string in dependency_strings:
- yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint)
+ try:
@@ -357,90 +362,14 @@ index 8c4e981..862be14 100644
+ yield InstallRequirement.from_line(_dependency_string, constraint=ireq.constraint)
+ except InvalidMarker:
+ yield InstallRequirement.from_line(dependency_string, constraint=ireq.constraint)
+
def reverse_dependencies(self, ireqs):
non_editable = [ireq for ireq in ireqs if not ireq.editable]
diff --git a/pipenv/patched/piptools/scripts/compile.py b/pipenv/patched/piptools/scripts/compile.py
index b41f8b2..0e83bfc 100644
--- a/pipenv/patched/piptools/scripts/compile.py
+++ b/pipenv/patched/piptools/scripts/compile.py
@@ -7,8 +7,8 @@ import os
import sys
import tempfile
-import pip
-from pip.req import InstallRequirement, parse_requirements
+import pip9
+from pip9.req import InstallRequirement, parse_requirements
from .. import click
from ..exceptions import PipToolsError
@@ -21,7 +21,7 @@ from ..writer import OutputWriter
DEFAULT_REQUIREMENTS_FILE = 'requirements.in'
-class PipCommand(pip.basecommand.Command):
+class PipCommand(pip9.basecommand.Command):
name = 'PipCommand'
@@ -251,8 +251,8 @@ def get_pip_command():
# General options (find_links, index_url, extra_index_url, trusted_host,
# and pre) are defered to pip.
pip_command = PipCommand()
- index_opts = pip.cmdoptions.make_option_group(
- pip.cmdoptions.index_group,
+ index_opts = pip9.cmdoptions.make_option_group(
+ pip9.cmdoptions.index_group,
pip_command.parser,
)
pip_command.parser.insert_option_group(0, index_opts)
diff --git a/pipenv/patched/piptools/scripts/sync.py b/pipenv/patched/piptools/scripts/sync.py
index 0f74370..c43c5af 100644
--- a/pipenv/patched/piptools/scripts/sync.py
+++ b/pipenv/patched/piptools/scripts/sync.py
@@ -5,7 +5,7 @@ from __future__ import (absolute_import, division, print_function,
import os
import sys
-import pip
+import pip9
from .. import click, sync
from ..exceptions import PipToolsError
@@ -45,7 +45,7 @@ def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet,
log.error('ERROR: ' + msg)
sys.exit(2)
- requirements = flat_map(lambda src: pip.req.parse_requirements(src, session=True),
+ requirements = flat_map(lambda src: pip9.req.parse_requirements(src, session=True),
src_files)
try:
@@ -54,7 +54,7 @@ def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet,
log.error(str(e))
sys.exit(2)
- installed_dists = pip.get_installed_distributions(skip=[], user_only=user_only)
+ installed_dists = pip9.get_installed_distributions(skip=[], user_only=user_only)
to_install, to_uninstall = sync.diff(requirements, installed_dists)
install_flags = []
diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py
index c67d1b5..db8bb9b 100644
index fde5816..d76695d 100644
--- a/pipenv/patched/piptools/utils.py
+++ b/pipenv/patched/piptools/utils.py
@@ -6,7 +6,7 @@ import sys
from itertools import chain, groupby
from collections import OrderedDict
-from pip.req import InstallRequirement
+from pip9.req import InstallRequirement
from first import first
@@ -41,16 +41,21 @@ def comment(text):
@@ -43,16 +43,21 @@ def comment(text):
return style(text, fg='green')
@@ -466,7 +395,7 @@ index c67d1b5..db8bb9b 100644
def format_requirement(ireq, marker=None):
@@ -64,7 +69,7 @@ def format_requirement(ireq, marker=None):
@@ -66,7 +71,7 @@ def format_requirement(ireq, marker=None):
line = str(ireq.req).lower()
if marker:
@@ -475,4 +404,77 @@ index c67d1b5..db8bb9b 100644
return line
diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py
index 7e8cdf3..96c8a1e 100644
--- a/piptools/_compat/pip_compat.py
+++ b/piptools/_compat/pip_compat.py
@@ -1,30 +1,40 @@
# -*- coding=utf-8 -*-
import importlib
-def do_import(module_path, subimport=None, old_path=None):
+
+def do_import(module_path, subimport=None, old_path=None, vendored_name=None):
internal = 'pip._internal.{0}'.format(module_path)
old_path = old_path or module_path
pip9 = 'pip.{0}'.format(old_path)
- try:
- _tmp = importlib.import_module(internal)
- except ImportError:
- _tmp = importlib.import_module(pip9)
+ _tmp = None
+ if vendored_name:
+ vendor = '{0}.{1}'.format(vendored_name, old_path)
+ try:
+ _tmp = importlib.import_module(vendor)
+ except ImportError:
+ pass
+ if not _tmp:
+ try:
+ _tmp = importlib.import_module(internal)
+ except ImportError:
+ _tmp = importlib.import_module(pip9)
if subimport:
return getattr(_tmp, subimport, _tmp)
return _tmp
-InstallRequirement = do_import('req.req_install', 'InstallRequirement')
-parse_requirements = do_import('req.req_file', 'parse_requirements')
-RequirementSet = do_import('req.req_set', 'RequirementSet')
-user_cache_dir = do_import('utils.appdirs', 'user_cache_dir')
-FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH')
-is_file_url = do_import('download', 'is_file_url')
-url_to_path = do_import('download', 'url_to_path')
-PackageFinder = do_import('index', 'PackageFinder')
-FormatControl = do_import('index', 'FormatControl')
-Wheel = do_import('wheel', 'Wheel')
-Command = do_import('basecommand', 'Command')
-cmdoptions = do_import('cmdoptions')
-get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils')
-PyPI = do_import('models.index', 'PyPI')
+InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name='notpip')
+parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name='notpip')
+RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name='notpip')
+user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name='notpip')
+FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name='notpip')
+is_file_url = do_import('download', 'is_file_url', vendored_name='notpip')
+url_to_path = do_import('download', 'url_to_path', vendored_name='notpip')
+PackageFinder = do_import('index', 'PackageFinder', vendored_name='notpip')
+FormatControl = do_import('index', 'FormatControl', vendored_name='notpip')
+Wheel = do_import('wheel', 'Wheel', vendored_name='notpip')
+Command = do_import('basecommand', 'Command', vendored_name='pip9')
+cmdoptions = do_import('cmdoptions', vendored_name='pip9')
+get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name='pip9')
+PyPI = do_import('models.index', 'PyPI', vendored_name='notpip')
+SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip')
diff --git a/pipenv/patched/piptools/_compat/__init__.py b/piptools/_compat/__init__.py
index 674674a..4259028 100644
--- a/pipenv/patched/piptools/_compat/__init__.py
+++ b/pipenv/patched/piptools/_compat/__init__.py
@@ -27,4 +27,5 @@ from .pip_compat import (
cmdoptions,
get_installed_distributions,
PyPI,
+ SafeFileCache,
)
+11
View File
@@ -351,3 +351,14 @@ index c9ebbf1..728f5cc 100644
path_to_url(wheel_file))
assert req.link.is_wheel
# extract the wheel into the dir
diff --git a/pipenv/vendor/pip9/__main__.py b/pipenv/vendor/pip9/__main__.py
index 9849a65c..da041f92 100644
--- a/pipenv/vendor/pip9/__main__.py
+++ b/pipenv/vendor/pip9/__main__.py
@@ -16,4 +16,4 @@ if __package__ == '':
import pip9 # noqa
if __name__ == '__main__':
- sys.exit(pip.main())
+ sys.exit(pip9.main())
+1 -1
View File
@@ -152,7 +152,7 @@ def test_environment_variable_value_does_not_change_hash(PipenvInstance, pypi):
with open(p.pipfile_path, 'w') as f:
f.write("""
[[source]]
url = 'https://${PYPI_USERNAME}:${PYPI_PASSWORD}@pypi.python.org/simple'
url = 'https://${PYPI_USERNAME}:${PYPI_PASSWORD}@pypi.org/simple'
verify_ssl = true
name = 'pypi'
[requires]
+1 -1
View File
@@ -107,7 +107,7 @@ def test_install_named_index_alias(PipenvInstance, pypi):
with open(p.pipfile_path, 'w') as f:
contents = """
[[source]]
url = "https://pypi.python.org/simple"
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
+3 -3
View File
@@ -199,7 +199,7 @@ def test_private_index_skip_lock(PipenvInstance):
with open(p.pipfile_path, 'w') as f:
contents = """
[[source]]
url = "https://pypi.python.org/simple"
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
@@ -228,7 +228,7 @@ def test_private_index_lock_requirements(PipenvInstance):
with open(p.pipfile_path, 'w') as f:
contents = """
[[source]]
url = "https://pypi.python.org/simple"
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
@@ -246,7 +246,7 @@ requests = "*"
assert c.return_code == 0
c = p.pipenv('lock -r')
assert c.return_code == 0
assert '-i https://pypi.python.org/simple' in c.out.strip()
assert '-i https://pypi.org/simple' in c.out.strip()
assert '--extra-index-url https://test.pypi.org/simple' in c.out.strip()
+2 -2
View File
@@ -42,7 +42,7 @@ verify_ssl = false
name = "testindex"
[[source]]
url = "https://pypi.python.org/simple"
url = "https://pypi.org/simple"
verify_ssl = "true"
name = "pypi"
@@ -60,7 +60,7 @@ six = {{version = "*", index = "pypi"}}
assert c.return_code == 0
project = Project()
sources = [
['pypi', 'https://pypi.python.org/simple'],
['pypi', 'https://pypi.org/simple'],
['testindex', os.environ.get('PIPENV_TEST_INDEX')]
]
for src in sources:
+1 -1
View File
@@ -1,6 +1,6 @@
[[source]]
url = "https://pypi.python.org/simple"
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
+1 -1
View File
@@ -10,7 +10,7 @@
"sources": [
{
"name": "pypi",
"url": "https://pypi.python.org/simple",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
+8 -8
View File
@@ -389,26 +389,26 @@ twine = "*"
([{'url': 'https://test.example.com/simple', 'verify_ssl': False}],
['-i', 'https://test.example.com/simple', '--trusted-host', 'test.example.com']),
([{'url': "https://pypi.python.org/simple"},
([{'url': "https://pypi.org/simple"},
{'url': "https://custom.example.com/simple"}],
['-i', 'https://pypi.python.org/simple',
['-i', 'https://pypi.org/simple',
'--extra-index-url', 'https://custom.example.com/simple']),
([{'url': "https://pypi.python.org/simple"},
([{'url': "https://pypi.org/simple"},
{'url': "https://custom.example.com/simple", 'verify_ssl': False}],
['-i', 'https://pypi.python.org/simple',
['-i', 'https://pypi.org/simple',
'--extra-index-url', 'https://custom.example.com/simple',
'--trusted-host', 'custom.example.com']),
([{'url': "https://pypi.python.org/simple"},
([{'url': "https://pypi.org/simple"},
{'url': "https://user:password@custom.example.com/simple", 'verify_ssl': False}],
['-i', 'https://pypi.python.org/simple',
['-i', 'https://pypi.org/simple',
'--extra-index-url', 'https://user:password@custom.example.com/simple',
'--trusted-host', 'custom.example.com']),
([{'url': "https://pypi.python.org/simple"},
([{'url': "https://pypi.org/simple"},
{'url': "https://user:password@custom.example.com/simple",}],
['-i', 'https://pypi.python.org/simple',
['-i', 'https://pypi.org/simple',
'--extra-index-url', 'https://user:password@custom.example.com/simple',]),
],
)