kennethreitz/pipenv
1
0
Fork 0
mirror of https://github.com/kennethreitz/pipenv.git synced 2026-06-05 14:50:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Minor code cleanup

Browse Source 
Signed-off-by: Dan Ryan <dan@danryan.co>

Add pytz and certifi updates

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix nondeterministic resolution bug

- Update dependencies
- Fix some issues with test logic
- Update piptools patch

Signed-off-by: Dan Ryan <dan@danryan.co>

Update more packages

Signed-off-by: Dan Ryan <dan@danryan.co>

Update tests and utils

Signed-off-by: Dan Ryan <dan@danryan.co>

Still need to tackle last few failures

- this will seriously help with resolution issues

Add alembic new version

Signed-off-by: Dan Ryan <dan@danryan.co>
This commit is contained in:
Dan Ryan
2018-06-20 22:56:22 -04:00
parent 93aceb1cc5
commit c31a311dc7
38 changed files with 343 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
news/2384.bugfix
+1 -1
View File
@@ -1 +1 @@
Dependencies with markers that don't match the current environment will now be skipped during ``pipenv lock``.
Resolved a bug in our patched resolvers which could cause nondeterministic resolution failures in certain conditions.
news/2384.feature
+1
View File
@@ -0,0 +1 @@
Optimized hashing speed.
news/2384.trivial
+1
View File
@@ -0,0 +1 @@
Added pytz 2018.4 wheel for testing -- needed for dependency resolution.
pipenv/core.py
+3 -5
View File
@@ -47,13 +47,11 @@ from .utils import (
is_star,
rmtree,
split_argument,
extract_uri_from_vcs_dep,
fs_str,
clean_resolved_dep,
)
from ._compat import (
TemporaryDirectory,
vcs,
Path
)
from .import pep508checker, progress
@@ -103,7 +101,7 @@ if not PIPENV_HIDE_EMOJIS:
):
INSTALL_LABEL = '🎅 '
else:
INSTALL_LABEL = '🐍 '
INSTALL_LABEL = '🝝 '
INSTALL_LABEL2 = crayons.normal('☤ ', bold=True)
STARTING_LABEL = ' '
else:
@@ -1006,7 +1004,7 @@ def do_lock(
pre=False,
keep_outdated=False,
write=True,
pypi_mirror = None,
pypi_mirror=None,
):
"""Executes the freeze functionality."""
from .utils import get_vcs_deps
@@ -1382,7 +1380,7 @@ def pip_install(
selective_upgrade=False,
requirements_dir=None,
extra_indexes=None,
pypi_mirror = None,
pypi_mirror=None,
):
from notpip._internal import logger as piplogger
from notpip._vendor.pyparsing import ParseException
pipenv/patched/piptools/repositories/local.py
+1 -1
View File
@@ -56,7 +56,7 @@ class LocalRequirementsRepository(BaseRepository):
if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin):
project, version, _ = as_tuple(existing_pin)
return make_install_requirement(
project, version, ireq.extras, constraint=ireq.constraint
project, version, ireq.extras, constraint=ireq.constraint, markers=ireq.markers
)
else:
return self.repository.find_best_match(ireq, prereleases)
pipenv/patched/piptools/repositories/pypi.py
+82 -45
View File
@@ -24,14 +24,14 @@ from .._compat import (
from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier
from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier
from pipenv.patched.notpip._vendor.pyparsing import ParseException
from ..cache import CACHE_DIR
from pipenv.environments import PIPENV_CACHE_DIR
from ..exceptions import NoCandidateFound
from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
make_install_requirement, format_requirement, dedup)
make_install_requirement, format_requirement, dedup, clean_requires_python)
from .base import BaseRepository
@@ -165,21 +165,7 @@ class PyPIRepository(BaseRepository):
return ireq # return itself as the best match
py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3])))
all_candidates = []
for c in self.find_all_candidates(ireq.name):
if c.requires_python:
# Old specifications had people setting this to single digits
# which is effectively the same as '>=digit,<digit+1'
if c.requires_python.isdigit():
c.requires_python = '>={0},<{1}'.format(c.requires_python, int(c.requires_python) + 1)
try:
specifier_set = SpecifierSet(c.requires_python)
except InvalidSpecifier:
pass
else:
if not specifier_set.contains(py_version):
continue
all_candidates.append(c)
all_candidates = clean_requires_python(self.find_all_candidates(ireq.name))
candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
try:
@@ -284,6 +270,20 @@ class PyPIRepository(BaseRepository):
os.makedirs(download_dir)
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
# Collect setup_requires info from local eggs.
# Do this after we call the preparer on these reqs to make sure their
# egg info has been created
setup_requires = {}
dist = None
if ireq.editable:
try:
dist = ireq.get_dist()
if dist.has_metadata('requires.txt'):
setup_requires = self.finder.get_extras_links(
dist.get_metadata_lines('requires.txt')
)
except (TypeError, ValueError):
pass
try:
# Pip < 9 and below
@@ -320,7 +320,7 @@ class PyPIRepository(BaseRepository):
finder=self.finder,
session=self.session,
upgrade_strategy="to-satisfy-only",
force_reinstall=False,
force_reinstall=True,
ignore_dependencies=False,
ignore_requires_python=True,
ignore_installed=True,
@@ -330,33 +330,37 @@ class PyPIRepository(BaseRepository):
ignore_compatibility=False
)
self.resolver.resolve(reqset)
result = reqset.requirements.values()
result = set(reqset.requirements.values())
# Collect setup_requires info from local eggs.
# Do this after we call the preparer on these reqs to make sure their
# egg info has been created
setup_requires = {}
if ireq.editable:
# HACK: Sometimes the InstallRequirement doesn't properly get
# these values set on it during the resolution process. It's
# difficult to pin down what is going wrong. This fixes things.
if not getattr(ireq, 'version', None):
try:
dist = ireq.get_dist()
if dist.has_metadata('requires.txt'):
setup_requires = self.finder.get_extras_links(
dist.get_metadata_lines('requires.txt')
)
# HACK: Sometimes the InstallRequirement doesn't properly get
# these values set on it during the resolution process. It's
# difficult to pin down what is going wrong. This fixes things.
ireq.version = dist.version
ireq.project_name = dist.project_name
ireq.req = dist.as_requirement()
except (TypeError, ValueError):
dist = ireq.get_dist() if not dist else None
ireq.version = ireq.get_dist().version
except (ValueError, OSError, TypeError) as e:
pass
if not getattr(ireq, 'project_name', None):
try:
ireq.project_name = dist.project_name if dist else None
except (ValueError, TypeError) as e:
pass
if not getattr(ireq, 'req', None):
try:
ireq.req = dist.as_requirement() if dist else None
except (ValueError, TypeError) as e:
pass
# Convert setup_requires dict into a somewhat usable form.
if setup_requires:
for section in setup_requires:
python_version = section
not_python = not (section.startswith('[') and ':' in section)
# This is for cleaning up :extras: formatted markers
# by adding them to the results of the resolver
# since any such extra would have been returned as a result anyway
for value in setup_requires[section]:
# This is a marker.
if value.startswith('[') and ':' in value:
@@ -370,17 +374,45 @@ class PyPIRepository(BaseRepository):
try:
if not not_python:
result = result + [InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))]
# Anything could go wrong here — can't be too careful.
# Anything could go wrong here -- can't be too careful.
except Exception:
pass
# this section properly creates 'python_version' markers for cross-python
# virtualenv creation and for multi-python compatibility.
requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else self.resolver.requires_python
if requires_python:
marker = 'python_version=="{0}"'.format(requires_python.replace(' ', ''))
new_req = InstallRequirement.from_line('{0}; {1}'.format(str(ireq.req), marker))
result = [new_req]
marker_str = ''
# This corrects a logic error from the previous code which said that if
# we Encountered any 'requires_python' attributes, basically only create a
# single result no matter how many we resolved. This should fix
# a majority of the remaining non-deterministic resolution issues.
if any(requires_python.startswith(op) for op in Specifier._operators.keys()):
# We are checking first if we have leading specifier operator
# if not, we can assume we should be doing a == comparison
specifierset = list(SpecifierSet(requires_python))
# for multiple specifiers, the correct way to represent that in
# a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
first_spec, marker_str = specifierset[0]._spec
if len(specifierset) > 1:
marker_str = [marker_str,]
for spec in specifierset[1:]:
marker_str.append(str(spec))
marker_str = ','.join(marker_str)
# join the leading specifier operator and the rest of the specifiers
marker_str = '{0}"{1}"'.format(first_spec, marker_str)
else:
marker_str = '=="{0}"'.format(requires_python.replace(' ', ''))
# The best way to add markers to a requirement is to make a separate requirement
# with only markers on it, and then to transfer the object istelf
marker_to_add = Requirement('fakepkg; python_version{0}'.format(marker_str)).marker
result.remove(ireq)
ireq.req.marker = marker_to_add
result.add(ireq)
self._dependencies_cache[ireq] = result
reqset.cleanup_files()
return set(self._dependencies_cache[ireq])
def get_hashes(self, ireq):
@@ -399,11 +431,16 @@ class PyPIRepository(BaseRepository):
# We need to get all of the candidates that match our current version
# pin, these will represent all of the files that could possibly
# satisfy this constraint.
all_candidates = self.find_all_candidates(ireq.name)
candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
matching_versions = list(
ireq.specifier.filter((candidate.version for candidate in all_candidates)))
matching_candidates = candidates_by_version[matching_versions[0]]
### Modification -- this is much more efficient....
### modification again -- still more efficient
matching_candidates = (
c for c in clean_requires_python(self.find_all_candidates(ireq.name))
if c.version in ireq.specifier
)
# candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
# matching_versions = list(
# ireq.specifier.filter((candidate.version for candidate in all_candidates)))
# matching_candidates = candidates_by_version[matching_versions[0]]
return {
self._hash_cache.get_hash(candidate.location)
pipenv/patched/piptools/utils.py
+18 -1
View File
@@ -11,13 +11,30 @@ from contextlib import contextmanager
from ._compat import InstallRequirement
from first import first
from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier
from .click import style
UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'}
def clean_requires_python(candidates):
"""Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes."""
all_candidates = []
for c in candidates:
if c.requires_python:
# Old specifications had people setting this to single digits
# which is effectively the same as '>=digit,<digit+1'
if c.requires_python.isdigit():
c.requires_python = '>={0},<{1}'.format(c.requires_python, int(c.requires_python) + 1)
try:
specifier_set = SpecifierSet(c.requires_python)
except InvalidSpecifier:
pass
all_candidates.append(c)
return all_candidates
def key_from_ireq(ireq):
"""Get a standardized key for an InstallRequirement."""
if ireq.req is None and ireq.link is not None:
pipenv/utils.py
+74 -32
View File
@@ -220,7 +220,9 @@ def prepare_pip_source_args(sources, pip_args=None):
def actually_resolve_deps(
deps, index_lookup, markers_lookup, project, sources, verbose, clear, pre, req_dir=None
):
from .vendor.packaging.markers import default_environment
from .patched.notpip._internal import basecommand
from .patched.notpip._internal.cmdoptions import no_binary, only_binary
from .patched.notpip._internal.req import parse_requirements
from .patched.notpip._internal.exceptions import DistributionNotFound
from .patched.notpip._vendor.requests.exceptions import HTTPError
@@ -248,38 +250,50 @@ def actually_resolve_deps(
dep, url = dep.split(' -i ')
req = Requirement.from_line(dep)
# Just use req.ireq directly, no need to actually use the extra file intermediares
# And also allows us to eliminate any markers that don't match before we start
# locking / resolving
ireq = req.ireq
if not ireq.markers or ireq.markers.evaluate():
constraints.append(req.ireq)
# extra_constraints = []
if url:
index_lookup[req.name] = project.get_source(url=url).get('name')
# strip the marker and re-add it later after resolution
# but we will need a fallback in case resolution fails
# eg pypiwin32
if req.markers:
markers_lookup[req.name] = req.markers.replace('"', "'")
constraints.append(req.constraint_line)
pip_command = get_pip_command()
constraints_file = None
pip_args = []
if sources:
pip_args = prepare_pip_source_args(sources, pip_args)
if verbose:
print('Using pip: {0}'.format(' '.join(pip_args)))
pip_args = pip_args.extend(['--cache-dir', PIPENV_CACHE_DIR])
pip_options, _ = pip_command.parse_args(pip_args)
with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f:
if sources:
requirementstxt_sources = ' '.join(pip_args) if pip_args else ''
requirementstxt_sources = requirementstxt_sources.replace(' --', '\n--')
f.write(u'{0}\n'.format(requirementstxt_sources))
f.write(u'\n'.join([_constraint for _constraint in constraints]))
constraints_file = f.name
pip_options, _ = pip_command.parser.parse_args(pip_args)
session = pip_command._build_session(pip_options)
pypi = PyPIRepository(
pip_options=pip_options, use_json=False, session=session
)
constraints = parse_requirements(constraints_file, finder=pypi.finder, session=pypi.session, options=pip_options)
constraints = [c for c in constraints]
if verbose:
logging.log.verbose = True
piptools_logging.log.verbose = True
resolved_tree = set()
resolver = Resolver(constraints=constraints, repository=pypi, clear_caches=clear, prereleases=pre)
# pre-resolve instead of iterating to avoid asking pypi for hashes of editable packages
hashes = None
try:
resolved_tree.update(resolver.resolve(max_rounds=PIPENV_MAX_ROUNDS))
results = resolver.resolve(max_rounds=PIPENV_MAX_ROUNDS)
hashes = resolver.resolve_hashes(results)
resolved_tree.update(results)
except (NoCandidateFound, DistributionNotFound, HTTPError) as e:
click_echo(
'{0}: Your dependencies could not be resolved. You likely have a '
@@ -307,8 +321,7 @@ def actually_resolve_deps(
raise RuntimeError
if cleanup_req_dir:
req_dir.cleanup()
return resolved_tree, resolver
return (resolved_tree, hashes, markers_lookup, resolver)
def venv_resolve_deps(
@@ -360,7 +373,7 @@ def resolve_deps(
python=False,
clear=False,
pre=False,
allow_global=False,
allow_global=False
):
"""Given a list of dependencies, return a resolved list of dependencies,
using pip-tools -- and their hashes, using the warehouse API / pip.
@@ -378,7 +391,7 @@ def resolve_deps(
req_dir = TemporaryDirectory(prefix='pipenv-', suffix='-requirements')
with HackedPythonVersion(python_version=python, python_path=python_path):
try:
resolved_tree, resolver = actually_resolve_deps(
resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps(
deps,
index_lookup,
markers_lookup,
@@ -387,7 +400,7 @@ def resolve_deps(
verbose,
clear,
pre,
req_dir=req_dir,
req_dir=req_dir
)
except RuntimeError:
# Don't exit here, like usual.
@@ -401,7 +414,7 @@ def resolve_deps(
try:
# Attempt to resolve again, with different Python version information,
# particularly for particularly particular packages.
resolved_tree, resolver = actually_resolve_deps(
resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps(
deps,
index_lookup,
markers_lookup,
@@ -410,7 +423,7 @@ def resolve_deps(
verbose,
clear,
pre,
req_dir=req_dir,
req_dir=req_dir
)
except RuntimeError:
req_dir.cleanup()
@@ -429,7 +442,9 @@ def resolve_deps(
else:
markers = markers_lookup.get(result.name)
collected_hashes = []
if any('python.org' in source['url'] or 'pypi.org' in source['url']
if result in hashes:
collected_hashes = list(hashes.get(result))
elif any('python.org' in source['url'] or 'pypi.org' in source['url']
for source in sources):
pkg_url = 'https://pypi.org/pypi/{0}/json'.format(name)
session = _get_requests_session()
@@ -453,14 +468,14 @@ def resolve_deps(
crayons.red('Warning', bold=True), name
)
)
# Collect un-collectable hashes (should work with devpi).
try:
collected_hashes = collected_hashes + list(
list(resolver.resolve_hashes([result]).items())[0][1]
)
except (ValueError, KeyError, ConnectionError, IndexError):
if verbose:
print('Error generating hash for {}'.format(name))
# # Collect un-collectable hashes (should work with devpi).
# try:
# collected_hashes = collected_hashes + list(
# list(resolver.resolve_hashes([result]).items())[0][1]
# )
# except (ValueError, KeyError, ConnectionError, IndexError):
# if verbose:
# print('Error generating hash for {}'.format(name))
collected_hashes = sorted(set(collected_hashes))
d = {'name': name, 'version': version, 'hashes': collected_hashes}
if index:
@@ -1182,9 +1197,34 @@ def get_vcs_deps(
return reqs, lockfile
def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None):
def translate_markers(pipfile_entry):
"""Take a pipfile entry and normalize its markers
Provide a pipfile entry which may have 'markers' as a key or it may have
any valid key from `packaging.markers.marker_context.keys()` and standardize
the format into {'markers': 'key == "some_value"'}.
:param pipfile_entry: A dictionariy of keys and values representing a pipfile entry
:type pipfile_entry: dict
:returns: A normalized dictionary with cleaned marker entries
"""
if not isinstance(pipfile_entry, Mapping):
raise TypeError('Entry is not a pipfile formatted mapping.')
from notpip._vendor.distlib.markers import DEFAULT_CONTEXT as marker_context
allowed_marker_keys = ['markers'] + [k for k in marker_context.keys()]
provided_keys = list(pipfile_entry.keys()) if hasattr(pipfile_entry, 'keys') else []
pipfile_marker = next((k for k in provided_keys if k in allowed_marker_keys), None)
new_pipfile = pipfile_entry.copy()
if pipfile_marker:
entry = "{0}".format(pipfile_entry[pipfile_marker])
if pipfile_marker != 'markers':
entry = "{0} {1}".format(pipfile_marker, entry)
new_pipfile.pop(pipfile_marker)
new_pipfile['markers'] = entry
return new_pipfile
def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None):
name = pep423_name(dep['name'])
# We use this to determine if there are any markers on top level packages
# So we can make sure those win out during resolution if the packages reoccur
@@ -1213,16 +1253,18 @@ def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None):
if 'markers' in dep:
# First, handle the case where there is no top level dependency in the pipfile
if not is_top_level:
lockfile['markers'] = dep['markers']
try:
lockfile['markers'] = translate_markers(dep)['markers']
except TypeError:
pass
# otherwise make sure we are prioritizing whatever the pipfile says about the markers
# If the pipfile says nothing, then we should put nothing in the lockfile
else:
pipfile_marker = next((k for k in dep_keys if k in allowed_marker_keys), None)
if pipfile_marker:
entry = "{0}".format(pipfile_entry[pipfile_marker])
if pipfile_marker != 'markers':
entry = "{0} {1}".format(pipfile_marker, entry)
lockfile['markers'] = entry
try:
pipfile_entry = translate_markers(pipfile_entry)
lockfile['markers'] = pipfile_entry.get('markers')
except TypeError:
pass
return {name: lockfile}
tasks/vendoring/patches/patched/piptools.patch
+154 -49
View File
@@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644
# NOTE
# We used to store the cache dir under ~/.pip-tools, which is not the
diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
index 1c4b943..7c6521d 100644
index 1c4b943..c645e08 100644
--- a/pipenv/patched/piptools/repositories/pypi.py
+++ b/pipenv/patched/piptools/repositories/pypi.py
@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function,
@@ -43,7 +43,7 @@ index 1c4b943..7c6521d 100644
+from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
+from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
+from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier
+from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier
+from pip._vendor.pyparsing import ParseException
+
from ..cache import CACHE_DIR
@@ -52,7 +52,7 @@ index 1c4b943..7c6521d 100644
-from ..utils import (fs_str, is_pinned_requirement, lookup_table,
- make_install_requirement)
+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
+ make_install_requirement, format_requirement, dedup)
+ make_install_requirement, format_requirement, dedup, clean_requires_python)
+
from .base import BaseRepository
@@ -130,27 +130,13 @@ index 1c4b943..7c6521d 100644
def freshen_build_caches(self):
"""
@@ -114,10 +164,29 @@ class PyPIRepository(BaseRepository):
@@ -114,10 +164,15 @@ class PyPIRepository(BaseRepository):
if ireq.editable:
return ireq # return itself as the best match
- all_candidates = self.find_all_candidates(ireq.name)
+ py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3])))
+ all_candidates = []
+ for c in self.find_all_candidates(ireq.name):
+ if c.requires_python:
+ # Old specifications had people setting this to single digits
+ # which is effectively the same as '>=digit,<digit+1'
+ if c.requires_python.isdigit():
+ c.requires_python = '>={0},<{1}'.format(c.requires_python, int(c.requires_python) + 1)
+ try:
+ specifier_set = SpecifierSet(c.requires_python)
+ except InvalidSpecifier:
+ pass
+ else:
+ if not specifier_set.contains(py_version):
+ continue
+ all_candidates.append(c)
+ all_candidates = clean_requires_python(self.find_all_candidates(ireq.name))
+
candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
- matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
@@ -162,7 +148,7 @@ index 1c4b943..7c6521d 100644
# Reuses pip's internal candidate sort key to sort
matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
@@ -126,11 +195,71 @@ class PyPIRepository(BaseRepository):
@@ -126,11 +181,71 @@ class PyPIRepository(BaseRepository):
best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)
# Turn the candidate into a pinned InstallRequirement
@@ -237,6 +223,27 @@ index 1c4b943..7c6521d 100644
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
@@ -155,6 +270,20 @@ class PyPIRepository(BaseRepository):
os.makedirs(download_dir)
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
+ # Collect setup_requires info from local eggs.
+ # Do this after we call the preparer on these reqs to make sure their
+ # egg info has been created
+ setup_requires = {}
+ dist = None
+ if ireq.editable:
+ try:
+ dist = ireq.get_dist()
+ if dist.has_metadata('requires.txt'):
+ setup_requires = self.finder.get_extras_links(
+ dist.get_metadata_lines('requires.txt')
+ )
+ except (TypeError, ValueError):
+ pass
try:
# Pip < 9 and below
@@ -164,11 +293,14 @@ class PyPIRepository(BaseRepository):
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
@@ -254,9 +261,12 @@ index 1c4b943..7c6521d 100644
)
except TypeError:
# Pip >= 10 (new resolver!)
@@ -190,14 +322,64 @@ class PyPIRepository(BaseRepository):
@@ -188,17 +320,99 @@ class PyPIRepository(BaseRepository):
finder=self.finder,
session=self.session,
upgrade_strategy="to-satisfy-only",
force_reinstall=False,
- force_reinstall=False,
+ force_reinstall=True,
ignore_dependencies=False,
- ignore_requires_python=False,
+ ignore_requires_python=True,
@@ -268,33 +278,37 @@ index 1c4b943..7c6521d 100644
)
self.resolver.resolve(reqset)
- self._dependencies_cache[ireq] = reqset.requirements.values()
+ result = reqset.requirements.values()
+ result = set(reqset.requirements.values())
+
+ # Collect setup_requires info from local eggs.
+ # Do this after we call the preparer on these reqs to make sure their
+ # egg info has been created
+ setup_requires = {}
+ if ireq.editable:
+ # HACK: Sometimes the InstallRequirement doesn't properly get
+ # these values set on it during the resolution process. It's
+ # difficult to pin down what is going wrong. This fixes things.
+ if not getattr(ireq, 'version', None):
+ try:
+ dist = ireq.get_dist()
+ if dist.has_metadata('requires.txt'):
+ setup_requires = self.finder.get_extras_links(
+ dist.get_metadata_lines('requires.txt')
+ )
+ # HACK: Sometimes the InstallRequirement doesn't properly get
+ # these values set on it during the resolution process. It's
+ # difficult to pin down what is going wrong. This fixes things.
+ ireq.version = dist.version
+ ireq.project_name = dist.project_name
+ ireq.req = dist.as_requirement()
+ except (TypeError, ValueError):
+ dist = ireq.get_dist() if not dist else None
+ ireq.version = ireq.get_dist().version
+ except (ValueError, OSError, TypeError) as e:
+ pass
+ if not getattr(ireq, 'project_name', None):
+ try:
+ ireq.project_name = dist.project_name if dist else None
+ except (ValueError, TypeError) as e:
+ pass
+ if not getattr(ireq, 'req', None):
+ try:
+ ireq.req = dist.as_requirement() if dist else None
+ except (ValueError, TypeError) as e:
+ pass
+
+ # Convert setup_requires dict into a somewhat usable form.
+ if setup_requires:
+ for section in setup_requires:
+ python_version = section
+ not_python = not (section.startswith('[') and ':' in section)
+
+ # This is for cleaning up :extras: formatted markers
+ # by adding them to the results of the resolver
+ # since any such extra would have been returned as a result anyway
+ for value in setup_requires[section]:
+ # This is a marker.
+ if value.startswith('[') and ':' in value:
@@ -308,21 +322,67 @@ index 1c4b943..7c6521d 100644
+ try:
+ if not not_python:
+ result = result + [InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))]
+ # Anything could go wrong here — can't be too careful.
+ # Anything could go wrong here -- can't be too careful.
+ except Exception:
+ pass
+
+ # this section properly creates 'python_version' markers for cross-python
+ # virtualenv creation and for multi-python compatibility.
+ requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else self.resolver.requires_python
+ if requires_python:
+ marker = 'python_version=="{0}"'.format(requires_python.replace(' ', ''))
+ new_req = InstallRequirement.from_line('{0}; {1}'.format(str(ireq.req), marker))
+ result = [new_req]
+ marker_str = ''
+ # This corrects a logic error from the previous code which said that if
+ # we Encountered any 'requires_python' attributes, basically only create a
+ # single result no matter how many we resolved. This should fix
+ # a majority of the remaining non-deterministic resolution issues.
+ if any(requires_python.startswith(op) for op in Specifier._operators.keys()):
+ # We are checking first if we have leading specifier operator
+ # if not, we can assume we should be doing a == comparison
+ specifierset = list(SpecifierSet(requires_python))
+ # for multiple specifiers, the correct way to represent that in
+ # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
+ first_spec, marker_str = specifierset[0]._spec
+ if len(specifierset) > 1:
+ marker_str = [marker_str,]
+ for spec in specifierset[1:]:
+ marker_str.append(str(spec))
+ marker_str = ','.join(marker_str)
+ # join the leading specifier operator and the rest of the specifiers
+ marker_str = '{0}"{1}"'.format(first_spec, marker_str)
+ else:
+ marker_str = '=="{0}"'.format(requires_python.replace(' ', ''))
+ # The best way to add markers to a requirement is to make a separate requirement
+ # with only markers on it, and then to transfer the object istelf
+ marker_to_add = Requirement('fakepkg; python_version{0}'.format(marker_str)).marker
+ result.remove(ireq)
+ ireq.req.marker = marker_to_add
+ result.add(ireq)
+
+ self._dependencies_cache[ireq] = result
reqset.cleanup_files()
+
return set(self._dependencies_cache[ireq])
@@ -224,17 +406,10 @@ class PyPIRepository(BaseRepository):
matching_candidates = candidates_by_version[matching_versions[0]]
def get_hashes(self, ireq):
@@ -217,24 +431,22 @@ class PyPIRepository(BaseRepository):
# We need to get all of the candidates that match our current version
# pin, these will represent all of the files that could possibly
# satisfy this constraint.
- all_candidates = self.find_all_candidates(ireq.name)
- candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
- matching_versions = list(
- ireq.specifier.filter((candidate.version for candidate in all_candidates)))
- matching_candidates = candidates_by_version[matching_versions[0]]
+ ### Modification -- this is much more efficient....
+ ### modification again -- still more efficient
+ matching_candidates = (
+ c for c in clean_requires_python(self.find_all_candidates(ireq.name))
+ if c.version in ireq.specifier
+ )
+ # candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
+ # matching_versions = list(
+ # ireq.specifier.filter((candidate.version for candidate in all_candidates)))
+ # matching_candidates = candidates_by_version[matching_versions[0]]
return {
- self._get_file_hash(candidate.location)
@@ -462,11 +522,56 @@ index 05ec8fd..c5eb728 100644
def reverse_dependencies(self, ireqs):
non_editable = [ireq for ireq in ireqs if not ireq.editable]
diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py
index 08dabe1..480ad1e 100644
--- a/pipenv/patched/piptools/repositories/local.py
+++ b/pipenv/patched/piptools/repositories/local.py
@@ -56,7 +56,7 @@ class LocalRequirementsRepository(BaseRepository):
if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin):
project, version, _ = as_tuple(existing_pin)
return make_install_requirement(
- project, version, ireq.extras, constraint=ireq.constraint
+ project, version, ireq.extras, constraint=ireq.constraint, markers=ireq.markers
)
else:
return self.repository.find_best_match(ireq, prereleases)
diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py
index fde5816..1d732bf 100644
index fde5816..5827a55 100644
--- a/pipenv/patched/piptools/utils.py
+++ b/pipenv/patched/piptools/utils.py
@@ -43,16 +43,51 @@ def comment(text):
@@ -11,13 +11,30 @@ from contextlib import contextmanager
from ._compat import InstallRequirement
from first import first
-
+from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier
from .click import style
UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'}
+def clean_requires_python(candidates):
+ """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes."""
+ all_candidates = []
+ for c in candidates:
+ if c.requires_python:
+ # Old specifications had people setting this to single digits
+ # which is effectively the same as '>=digit,<digit+1'
+ if c.requires_python.isdigit():
+ c.requires_python = '>={0},<{1}'.format(c.requires_python, int(c.requires_python) + 1)
+ try:
+ specifier_set = SpecifierSet(c.requires_python)
+ except InvalidSpecifier:
+ pass
+ all_candidates.append(c)
+ return all_candidates
+
+
def key_from_ireq(ireq):
"""Get a standardized key for an InstallRequirement."""
if ireq.req is None and ireq.link is not None:
@@ -43,16 +60,51 @@ def comment(text):
return style(text, fg='green')
@@ -522,7 +627,7 @@ index fde5816..1d732bf 100644
def format_requirement(ireq, marker=None):
@@ -63,10 +98,10 @@ def format_requirement(ireq, marker=None):
@@ -63,10 +115,10 @@ def format_requirement(ireq, marker=None):
if ireq.editable:
line = '-e {}'.format(ireq.link)
else:
tests/integration/test_install_uri.py
+2 -2
View File
@@ -138,8 +138,8 @@ def test_install_editable_git_tag(PipenvInstance, pip_src_dir, pypi):
@pytest.mark.install
@pytest.mark.index
@pytest.mark.needs_internet
def test_install_named_index_alias(PipenvInstance, pypi):
with PipenvInstance(pypi=pypi) as p:
def test_install_named_index_alias(PipenvInstance):
with PipenvInstance() as p:
with open(p.pipfile_path, 'w') as f:
contents = """
[[source]]
tests/integration/test_lock.py
+3 -3
View File
@@ -262,8 +262,8 @@ def test_private_index_mirror_lock_requirements(PipenvInstance):
with temp_environ(), PipenvInstance(chdir=True) as p:
# Using pypi.python.org as pipenv-test-public-package is not
# included in the local pypi mirror
mirror_url = "https://pypi.python.org/simple"
os.environ.pop('PIPENV_TEST_INDEX', None)
mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.kennethreitz.org/simple")
# os.environ.pop('PIPENV_TEST_INDEX', None)
with open(p.pipfile_path, 'w') as f:
contents = """
[[source]]
@@ -277,7 +277,7 @@ verify_ssl = true
name = "testpypi"
[packages]
pipenv-test-private-package = {version = "*", index = "testpypi"}
six = {version = "*", index = "testpypi"}
requests = "*"
""".strip()
f.write(contents)
tests/integration/test_pipenv.py
+2 -2
View File
@@ -74,8 +74,8 @@ def test_update_locks(PipenvInstance, pypi):
fh.write(pipfile_contents)
c = p.pipenv('update requests')
assert c.return_code == 0
assert p.lockfile['default']['requests']['version'] == '==2.18.4'
assert p.lockfile['default']['requests']['version'] == '==2.19.1'
c = p.pipenv('run pip freeze')
assert c.return_code == 0
lines = c.out.splitlines()
assert 'requests==2.18.4' in [l.strip() for l in lines]
assert 'requests==2.19.1' in [l.strip() for l in lines]
tests/integration/test_sync.py
+1 -1
View File
@@ -22,7 +22,7 @@ def test_sync_error_without_lockfile(PipenvInstance, pypi):
@pytest.mark.lock
def test_mirror_lock_sync(PipenvInstance, pypi):
with temp_environ(), PipenvInstance(chdir=True) as p:
mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.python.org/simple")
mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.kennethreitz..org/simple")
assert 'pypi.org' not in mirror_url
with open(p.pipfile_path, 'w') as f:
f.write("""
tests/pypi/alembic/alembic-0.9.9.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/certifi/certifi-2018.4.16-py2.py3-none-any.whl
BIN
View File
Binary file not shown.
tests/pypi/certifi/certifi-2018.4.16.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/idna/idna-2.7-py2.py3-none-any.whl
BIN
View File
Binary file not shown.
tests/pypi/idna/idna-2.7.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/jdcal/jdcal-1.4-py2.py3-none-any.whl
BIN
View File
Binary file not shown.
tests/pypi/jdcal/jdcal-1.4.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/maya/maya-0.5.0-py2.py3-none-any.whl
BIN
View File
Binary file not shown.
tests/pypi/maya/maya-0.5.0.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/openpyxl/openpyxl-2.5.4.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_i686.whl
BIN
View File
Binary file not shown.
tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_x86_64.whl
BIN
View File
Binary file not shown.
tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_i686.whl
BIN
View File
Binary file not shown.
tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_x86_64.whl
BIN
View File
Binary file not shown.
tests/pypi/pendulum/pendulum-1.5.1.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/python-dateutil/python-dateutil-2.7.3.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/python-dateutil/python_dateutil-2.7.3-py2.py3-none-any.whl
BIN
View File
Binary file not shown.
tests/pypi/pytz/pytz-2018.4.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/pytzdata/pytzdata-2018.5-py2.py3-none-any.whl
BIN
View File
Binary file not shown.
tests/pypi/pytzdata/pytzdata-2018.5.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/requests/requests-2.19.1-py2.py3-none-any.whl
BIN
View File
Binary file not shown.
tests/pypi/requests/requests-2.19.1.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/sqlalchemy/SQLAlchemy-1.2.8.tar.gz
BIN
View File
Binary file not shown.
tests/pypi/urllib3/urllib3-1.23-py2.py3-none-any.whl
BIN
View File
Binary file not shown.
tests/pypi/urllib3/urllib3-1.23.tar.gz
BIN
View File
Binary file not shown.