kennethreitz/pipenv
1
0
Fork 0
mirror of https://github.com/kennethreitz/pipenv.git synced 2026-06-05 22:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Patch piptools to use current environment python

Browse Source 
- Fixes #2088, #2234, #1901
- Fully leverage piptools' compile functionality by using constraints
  in the same `RequirementSet` during resolution
- Use `PIP_PYTHON_PATH` for compatibility check to filter out
  `requires_python` markers
- Fix vcs resolution
- Update JSON API endpoints
- Enhance resolution for editable dependencies
- Minor fix for adding packages to pipfiles

Signed-off-by: Dan Ryan <dan@danryan.co>
This commit is contained in:
Dan Ryan
2018-05-25 20:20:55 -04:00
parent 7abc2fd3a1
commit bba2f38d91
4 changed files with 105 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipenv/patched/piptools/repositories/pypi.py
+26 -10
View File
@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function,
import hashlib
import os
import sys
from contextlib import contextmanager
from shutil import rmtree
@@ -20,14 +21,17 @@ from .._compat import (
SafeFileCache,
)
from notpip._vendor.packaging.requirements import InvalidRequirement
from notpip._vendor.packaging.requirements import InvalidRequirement, Requirement
from notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
from notpip._vendor.packaging.specifiers import SpecifierSet
from notpip._vendor.pyparsing import ParseException
from ..cache import CACHE_DIR
from pipenv.environments import PIPENV_CACHE_DIR
from ..exceptions import NoCandidateFound
from ..utils import (fs_str, is_pinned_requirement, lookup_table,
make_install_requirement)
from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
make_install_requirement, format_requirement, dedup)
from .base import BaseRepository
@@ -159,7 +163,15 @@ class PyPIRepository(BaseRepository):
if ireq.editable:
return ireq # return itself as the best match
all_candidates = self.find_all_candidates(ireq.name)
_all_candidates = self.find_all_candidates(ireq.name)
all_candidates = []
py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3])))
for c in _all_candidates:
if c.requires_python:
python_specifier = SpecifierSet(c.requires_python)
if not python_specifier.contains(py_version):
continue
all_candidates.append(c)
candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
try:
matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
@@ -194,11 +206,12 @@ class PyPIRepository(BaseRepository):
r = self.session.get(url)
# TODO: Latest isn't always latest.
latest = list(r.json()['releases'].keys())[-1]
if str(ireq.req.specifier) == '=={0}'.format(latest):
latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest)
latest_requires = self.session.get(latest_url)
for requires in latest_requires.json().get('info', {}).get('requires_dist', {}):
releases = list(r.json()['releases'].keys())
match = [r for r in releases if '=={0}'.format(r) == str(ireq.req.specifier)]
if match:
release_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, match[0])
release_requires = self.session.get(release_url)
for requires in release_requires.json().get('info', {}).get('requires_dist', {}):
i = InstallRequirement.from_line(requires)
if 'extra' not in repr(i.markers):
@@ -245,7 +258,10 @@ class PyPIRepository(BaseRepository):
setup_requires = self.finder.get_extras_links(
dist.get_metadata_lines('requires.txt')
)
except TypeError:
ireq.version = dist.version
ireq.project_name = dist.project_name
ireq.req = dist.as_requirement()
except (TypeError, ValueError):
pass
if ireq not in self._dependencies_cache:
pipenv/project.py
+3 -3
View File
@@ -20,7 +20,7 @@ except ImportError:
from pathlib2 import Path
from .cmdparse import Script
from .vendor.requirementslib import Requirement
from .vendor.requirementslib.requirements import Requirement
from .utils import (
atomic_open_for_write,
mkdir_p,
@@ -728,8 +728,8 @@ class Project(object):
# Read and append Pipfile.
p = self.parsed_pipfile
# Don't re-capitalize file URLs or VCSs.
package = Requirement.from_line(package_name)
converted = first(package.as_pipfile().values())
package = Requirement.from_line(package_name.strip())
_, converted = package.pipfile_entry
key = 'dev-packages' if dev else 'packages'
# Set empty group if it doesn't exist yet.
if key not in p:
pipenv/utils.py
+24 -32
View File
@@ -221,6 +221,7 @@ def actually_resolve_reps(
):
from .patched.notpip._internal import basecommand
from .patched.notpip._internal.req import parse_requirements
from .patched.notpip._internal.req.req_install import InstallRequirement
from .patched.notpip._vendor import requests as pip_requests
from .patched.notpip._internal.exceptions import DistributionNotFound
from .patched.notpip._vendor.requests.exceptions import HTTPError
@@ -236,48 +237,37 @@ def actually_resolve_reps(
name = 'PipCommand'
constraints = []
tmpfile_constraints = []
cleanup_req_dir = False
if not req_dir:
req_dir = TemporaryDirectory(suffix='-requirements', prefix='pipenv-')
cleanup_req_dir = True
for dep in deps:
if dep:
if dep.startswith('-e '):
constraint = req.InstallRequirement.from_editable(
dep[len('-e '):]
)
else:
tmpfile_constraints.append(dep)
req = Requirement.from_line(dep)
# extra_constraints = []
url = None
if ' -i ' in dep:
index_lookup[req.name] = project.get_source(
url=dep.split(' -i ')[1]
).get(
'name'
)
if dep.markers:
markers_lookup[dep.name] = str(
dep.markers_as_pip
).replace(
'"', "'"
)
constraints.append(req)
dep, url = dep.split(' -i ')
req = Requirement.from_line(dep)
_line = req.as_line()
constraints.append(_line)
# extra_constraints = []
if url:
index_lookup[req.name] = project.get_source(url=url).get('name')
if req.markers:
markers_lookup[req.name] = req.markers_as_pip
constraints_file = None
with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f:
f.write('\n'.join(tmpfile_constraints))
constraints_file = f.name
pip_command = get_pip_command()
pip_args = []
if sources:
pip_args = prepare_pip_source_args(sources, pip_args)
with NamedTemporaryFile(mode='w', prefix='pipenv-', suffix='-constraints.txt', dir=req_dir.name, delete=False) as f:
f.write(u'\n'.join([_constraint for _constraint in constraints]))
constraints_file = f.name
if verbose:
print('Using pip: {0}'.format(' '.join(pip_args)))
pip_options, _ = pip_command.parse_args(pip_args)
session = pip_command._build_session(pip_options)
pypi = PyPIRepository(
pip_options=pip_options, use_json=False, session=session
pip_options=pip_options, use_json=True, session=session
)
if verbose:
logging.log.verbose = True
@@ -1138,7 +1128,7 @@ def install_or_update_vcs(vcs_obj, src_dir, name, rev=None):
def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False):
from ._compat import vcs
from .patched.notpip._internal.vcs import VcsSupport
section = 'vcs_dev_packages' if dev else 'vcs_packages'
lines = []
lockfiles = []
@@ -1146,7 +1136,7 @@ def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=Fals
packages = getattr(project, section)
except AttributeError:
return [], []
vcs_registry = vcs()
vcs_registry = VcsSupport
vcs_uri_map = {
extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')}
for k, v in packages.items()
@@ -1162,13 +1152,15 @@ def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=Fals
pipfile_rev = vcs_uri_map[_vcs_match]['ref']
src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src'))
mkdir_p(src_dir)
pipfile_req = Requirement.from_pipfile(pipfile_name, [], packages[pipfile_name])
names = {pipfile_name}
_pip_uri = line.lstrip('-e ')
backend_name = str(_pip_uri.split('+', 1)[0])
backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)]
__vcs = backend(url=_pip_uri)
backend = vcs_registry()._registry.get(pipfile_req.vcs)
# TODO: Why doesn't pip freeze list 'git+git://' formatted urls?
if line.startswith('-e ') and not '{0}+'.format(pipfile_req.vcs) in line:
line = line.replace('-e ', '-e {0}+'.format(pipfile_req.vcs))
installed = Requirement.from_line(line)
__vcs = backend(url=installed.req.uri)
names.add(installed.normalized_name)
locked_rev = None
for _name in names:
tasks/vendoring/patches/patched/piptools.patch
+52 -21
View File
@@ -19,10 +19,18 @@ index 4e6174c..75f9b49 100644
# NOTE
# We used to store the cache dir under ~/.pip-tools, which is not the
diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
index 1c4b943..8320e14 100644
index 1c4b943..858d697 100644
--- a/pipenv/patched/piptools/repositories/pypi.py
+++ b/pipenv/patched/piptools/repositories/pypi.py
@@ -15,10 +15,16 @@ from .._compat import (
@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function,
import hashlib
import os
+import sys
from contextlib import contextmanager
from shutil import rmtree
@@ -15,13 +16,22 @@ from .._compat import (
Wheel,
FAVORITE_HASH,
TemporaryDirectory,
@@ -32,15 +40,23 @@ index 1c4b943..8320e14 100644
+ SafeFileCache,
)
+from pip._vendor.packaging.requirements import InvalidRequirement
+from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
+from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
+from pip._vendor.packaging.specifiers import SpecifierSet
+from pip._vendor.pyparsing import ParseException
+
from ..cache import CACHE_DIR
+from pipenv.environments import PIPENV_CACHE_DIR
from ..exceptions import NoCandidateFound
from ..utils import (fs_str, is_pinned_requirement, lookup_table,
make_install_requirement)
@@ -37,6 +43,40 @@ except ImportError:
-from ..utils import (fs_str, is_pinned_requirement, lookup_table,
- make_install_requirement)
+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
+ make_install_requirement, format_requirement, dedup)
+
from .base import BaseRepository
@@ -37,6 +47,40 @@ except ImportError:
from pip.wheel import WheelCache
@@ -81,7 +97,7 @@ index 1c4b943..8320e14 100644
class PyPIRepository(BaseRepository):
DEFAULT_INDEX_URL = PyPI.simple_url
@@ -46,10 +86,11 @@ class PyPIRepository(BaseRepository):
@@ -46,10 +90,11 @@ class PyPIRepository(BaseRepository):
config), but any other PyPI mirror can be used if index_urls is
changed/configured on the Finder.
"""
@@ -95,7 +111,7 @@ index 1c4b943..8320e14 100644
index_urls = [pip_options.index_url] + pip_options.extra_index_urls
if pip_options.no_index:
@@ -74,11 +115,15 @@ class PyPIRepository(BaseRepository):
@@ -74,11 +119,15 @@ class PyPIRepository(BaseRepository):
# of all secondary dependencies for the given requirement, so we
# only have to go to disk once for each requirement
self._dependencies_cache = {}
@@ -113,9 +129,20 @@ index 1c4b943..8320e14 100644
def freshen_build_caches(self):
"""
@@ -116,8 +161,11 @@ class PyPIRepository(BaseRepository):
@@ -114,10 +163,21 @@ class PyPIRepository(BaseRepository):
if ireq.editable:
return ireq # return itself as the best match
all_candidates = self.find_all_candidates(ireq.name)
- all_candidates = self.find_all_candidates(ireq.name)
+ _all_candidates = self.find_all_candidates(ireq.name)
+ all_candidates = []
+ py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3])))
+ for c in _all_candidates:
+ if c.requires_python:
+ python_specifier = SpecifierSet(c.requires_python)
+ if not python_specifier.contains(py_version):
+ continue
+ all_candidates.append(c)
candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
- matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
+ try:
@@ -126,7 +153,7 @@ index 1c4b943..8320e14 100644
# Reuses pip's internal candidate sort key to sort
matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
@@ -126,11 +174,60 @@ class PyPIRepository(BaseRepository):
@@ -126,11 +186,61 @@ class PyPIRepository(BaseRepository):
best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)
# Turn the candidate into a pinned InstallRequirement
@@ -153,11 +180,12 @@ index 1c4b943..8320e14 100644
+ r = self.session.get(url)
+
+ # TODO: Latest isn't always latest.
+ latest = list(r.json()['releases'].keys())[-1]
+ if str(ireq.req.specifier) == '=={0}'.format(latest):
+ latest_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, latest)
+ latest_requires = self.session.get(latest_url)
+ for requires in latest_requires.json().get('info', {}).get('requires_dist', {}):
+ releases = list(r.json()['releases'].keys())
+ match = [r for r in releases if '=={0}'.format(r) == str(ireq.req.specifier)]
+ if match:
+ release_url = 'https://pypi.org/pypi/{0}/{1}/json'.format(ireq.req.name, match[0])
+ release_requires = self.session.get(release_url)
+ for requires in release_requires.json().get('info', {}).get('requires_dist', {}):
+ i = InstallRequirement.from_line(requires)
+
+ if 'extra' not in repr(i.markers):
@@ -190,7 +218,7 @@ index 1c4b943..8320e14 100644
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
@@ -139,6 +236,18 @@ class PyPIRepository(BaseRepository):
@@ -139,6 +249,21 @@ class PyPIRepository(BaseRepository):
if not (ireq.editable or is_pinned_requirement(ireq)):
raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq))
@@ -203,13 +231,16 @@ index 1c4b943..8320e14 100644
+ setup_requires = self.finder.get_extras_links(
+ dist.get_metadata_lines('requires.txt')
+ )
+ except TypeError:
+ ireq.version = dist.version
+ ireq.project_name = dist.project_name
+ ireq.req = dist.as_requirement()
+ except (TypeError, ValueError):
+ pass
+
if ireq not in self._dependencies_cache:
if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)):
# No download_dir for locally available editable requirements.
@@ -164,11 +273,14 @@ class PyPIRepository(BaseRepository):
@@ -164,11 +289,14 @@ class PyPIRepository(BaseRepository):
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
@@ -226,7 +257,7 @@ index 1c4b943..8320e14 100644
)
except TypeError:
# Pip >= 10 (new resolver!)
@@ -190,14 +302,44 @@ class PyPIRepository(BaseRepository):
@@ -190,14 +318,44 @@ class PyPIRepository(BaseRepository):
upgrade_strategy="to-satisfy-only",
force_reinstall=False,
ignore_dependencies=False,
@@ -273,7 +304,7 @@ index 1c4b943..8320e14 100644
reqset.cleanup_files()
return set(self._dependencies_cache[ireq])
@@ -224,17 +366,10 @@ class PyPIRepository(BaseRepository):
@@ -224,17 +382,10 @@ class PyPIRepository(BaseRepository):
matching_candidates = candidates_by_version[matching_versions[0]]
return {