kennethreitz/pipenv
1
0
Fork 0
mirror of https://github.com/kennethreitz/pipenv.git synced 2026-06-05 22:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update requirementslib and fix VCS installation

Browse Source 
Signed-off-by: Dan Ryan <dan@danryan.co>

Don't re-clone repos now that this works

Signed-off-by: Dan Ryan <dan@danryan.co>

Prune peeps directory from manifest

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix nonetype uris

Signed-off-by: Dan Ryan <dan@danryan.co>

Manually lock requirements?

Signed-off-by: Dan Ryan <dan@danryan.co>

Update requirementslib - leave context before updating sha

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix requirementslib vcs checkouts

Signed-off-by: Dan Ryan <dan@danryan.co>

fix tmpdir implementation

Signed-off-by: Dan Ryan <dan@danryan.co>

final fix for vcs uris

Signed-off-by: Dan Ryan <dan@danryan.co>

Allow for adding requirements objects directly to pipfile

Signed-off-by: Dan Ryan <dan@danryan.co>

Update piptools patch

Signed-off-by: Dan Ryan <dan@danryan.co>
This commit is contained in:
Dan Ryan
2018-09-03 19:03:57 -04:00
committed by Tzu-ping Chung
parent f6f98f4683
commit 3d33e5a257
9 changed files with 106 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
MANIFEST.in
+2
View File
@@ -24,6 +24,8 @@ recursive-include docs/_templates *.html
recursive-include docs/_static *.js *.css *.png
recursive-exclude docs requirements*.txt
prune peeps
prune .buildkite
prune .github
prune .vsts-ci
pipenv/_compat.py
+1 -1
View File
@@ -90,7 +90,7 @@ class TemporaryDirectory(object):
in it are removed.
"""
def __init__(self, suffix=None, prefix=None, dir=None):
def __init__(self, suffix="", prefix="", dir=None):
if "RAM_DISK" in os.environ:
import uuid
pipenv/core.py
+1 -1
View File
@@ -1908,7 +1908,7 @@ def do_install(
)
# Add the package to the Pipfile.
try:
project.add_package_to_pipfile(pkg_requirement.as_line(), dev)
project.add_package_to_pipfile(pkg_requirement, dev)
except ValueError as e:
click.echo(
"{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), e)
pipenv/project.py
+3 -2
View File
@@ -770,13 +770,14 @@ class Project(object):
del p[key][name]
self.write_toml(p)
def add_package_to_pipfile(self, package_name, dev=False):
def add_package_to_pipfile(self, package, dev=False):
from .vendor.requirementslib import Requirement
# Read and append Pipfile.
p = self.parsed_pipfile
# Don't re-capitalize file URLs or VCSs.
package = Requirement.from_line(package_name.strip())
if not isinstance(package, Requirement):
package = Requirement.from_line(package.strip())
_, converted = package.pipfile_entry
key = "dev-packages" if dev else "packages"
# Set empty group if it doesn't exist yet.
pipenv/utils.py
+12 -7
View File
@@ -1146,6 +1146,7 @@ def get_vcs_deps(
pypi_mirror=None,
):
from ._compat import TemporaryDirectory, Path
import atexit
from .vendor.requirementslib import Requirement
section = "vcs_dev_packages" if dev else "vcs_packages"
@@ -1160,12 +1161,18 @@ def get_vcs_deps(
os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src"))
)
src_dir.mkdir(mode=0o775, exist_ok=True)
else:
src_dir = TemporaryDirectory(prefix="pipenv-lock-dir")
atexit.register(src_dir.cleanup)
for pkg_name, pkg_pipfile in packages.items():
requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile)
name = requirement.normalized_name
commit_hash = None
if requirement.is_vcs:
lockfile[name] = requirement.pipfile_entry[1]
lockfile[name]['ref'] = requirement.req.repo.get_commit_hash()
with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo:
commit_hash = repo.get_commit_hash()
lockfile[name] = requirement.pipfile_entry[1]
lockfile[name]['ref'] = commit_hash
reqs.append(requirement)
return reqs, lockfile
@@ -1197,13 +1204,11 @@ def translate_markers(pipfile_entry):
for m in pipfile_markers:
entry = "{0}".format(pipfile_entry[m])
if m != "markers":
marker_set.add(str(Marker("{0}'{1}'".format(m, entry))))
marker_set.add(str(Marker("{0}{1}".format(m, entry))))
new_pipfile.pop(m)
marker_set.add(str(entry))
if marker_set:
new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s)
if " and " in s else s
for s in sorted(dedup(marker_set))])))
new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s) if " and " in s else s
for s in sorted(dedup(marker_set))]))).replace('"', "'")
return new_pipfile
pipenv/vendor/requirementslib/models/requirements.py
Vendored
+60 -31
View File
@@ -1,30 +1,35 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import atexit
import collections
import hashlib
import os
from contextlib import contextmanager
import attr
import atexit
from first import first
from packaging.markers import Marker
from packaging.requirements import Requirement as PackagingRequirement
from packaging.specifiers import Specifier, SpecifierSet
from packaging.utils import canonicalize_name
from six.moves.urllib import parse as urllib_parse
from six.moves.urllib.parse import unquote
from pip_shims.shims import (
InstallRequirement, Link, Wheel, _strip_extras, parse_version, path_to_url,
url_to_path
)
from six.moves.urllib import parse as urllib_parse
from six.moves.urllib.parse import unquote
from vistir.compat import FileNotFoundError, Path, TemporaryDirectory
from vistir.misc import dedup
from vistir.path import get_converted_relative_path, is_valid_url, is_file_url, mkdir_p
from vistir.path import (
create_tracked_tempdir, get_converted_relative_path, is_file_url,
is_valid_url, mkdir_p
)
from ..exceptions import RequirementError
from ..utils import VCS_LIST, is_vcs, is_installable_file
from ..utils import VCS_LIST, is_installable_file, is_vcs
from .baserequirement import BaseRequirement
from .dependencies import (
AbstractDependency, find_all_matches, get_abstract_dependencies,
@@ -32,15 +37,14 @@ from .dependencies import (
)
from .markers import PipenvMarkers
from .utils import (
HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, filter_none,
format_requirement, get_version, init_requirement,
HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string,
filter_none, format_requirement, get_version, init_requirement,
is_pinned_requirement, make_install_requirement, optional_instance_of,
parse_extras, specs_to_string, split_markers_from_line,
split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path,
validate_specifiers, validate_vcs, extras_to_string
validate_specifiers, validate_vcs
)
from .vcs import VCSRepository
from packaging.requirements import Requirement as PackagingRequirement
@attr.s
@@ -481,6 +485,9 @@ class VCSRequirement(FileRequirement):
req = attr.ib()
def __attrs_post_init__(self):
if not self.uri:
if self.path:
self.uri = path_to_url(self.path)
split = urllib_parse.urlsplit(self.uri)
scheme, rest = split[0], split[1:]
vcs_type = ""
@@ -493,9 +500,10 @@ class VCSRequirement(FileRequirement):
@link.default
def get_link(self):
uri = self.uri if self.uri else path_to_url(self.path)
return build_vcs_link(
self.vcs,
add_ssh_scheme_to_git_uri(self.uri),
add_ssh_scheme_to_git_uri(uri),
name=self.name,
ref=self.ref,
subdirectory=self.subdirectory,
@@ -564,21 +572,17 @@ class VCSRequirement(FileRequirement):
def get_checkout_dir(self, src_dir=None):
src_dir = os.environ.get('PIP_SRC', None) if not src_dir else src_dir
checkout_dir = None
if self.is_local and self.editable:
if self.is_local:
path = self.path
if not path:
path = url_to_path(self.uri)
if path and os.path.exists(path):
checkout_dir = Path(self.path).absolute().as_posix()
checkout_dir = os.path.abspath(path)
return checkout_dir
return src_dir
return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name)
def get_vcs_repo(self, src_dir=None):
checkout_dir = self.get_checkout_dir(src_dir=src_dir)
if not checkout_dir:
_src_dir = TemporaryDirectory()
atexit.register(_src_dir.cleanup)
checkout_dir = Path(_src_dir.name).joinpath(self.name).absolute().as_posix()
url = "{0}#egg={1}".format(self.vcs_uri, self.name)
vcsrepo = VCSRepository(
url=url,
@@ -587,25 +591,36 @@ class VCSRequirement(FileRequirement):
checkout_directory=checkout_dir,
vcs_type=self.vcs
)
if not (self.is_local and self.editable):
if not self.is_local:
vcsrepo.obtain()
return vcsrepo
def get_commit_hash(self):
hash_ = None
hash_ = self.repo.get_commit_hash()
return hash_
def update_repo(self, src_dir=None, ref=None):
ref = self.ref if not ref else ref
if not (self.is_local and self.editable):
self.repo.update()
if ref:
self.ref = ref
else:
if self.ref:
ref = self.ref
repo_hash = None
if not self.is_local and ref is not None:
self.repo.checkout_ref(ref)
hash_ = self.repo.get_commit_hash()
return hash_
repo_hash = self.repo.get_commit_hash()
return repo_hash
def lock_vcs_ref(self):
@contextmanager
def locked_vcs_repo(self, src_dir=None):
vcsrepo = self.get_vcs_repo(src_dir=src_dir)
if self.ref and not self.is_local:
vcsrepo.checkout_ref(self.ref)
self.ref = self.get_commit_hash()
self.req.revision = self.ref
yield vcsrepo
self._repo = vcsrepo
@classmethod
def from_pipfile(cls, name, pipfile):
@@ -627,12 +642,17 @@ class VCSRequirement(FileRequirement):
"{0}+{1}".format(key, pipfile.get(key))
).split("+", 1)[1]
url_keys = [pipfile.get(key), composed_uri]
is_url = any(validity_fn(url_key) for url_key in url_keys for validity_fn in [is_valid_url, is_file_url])
target_key = "uri" if is_url else "path"
creation_args[target_key] = pipfile.get(key)
if any(is_valid_url(k) for k in url_keys) or any(is_file_url(k) for k in url_keys):
creation_args["uri"] = pipfile.get(key)
else:
creation_args["path"] = pipfile.get(key)
if os.path.isabs(pipfile.get(key)):
creation_args["uri"] = Path(pipfile.get(key)).absolute().as_uri()
else:
creation_args[key] = pipfile.get(key)
creation_args["name"] = name
print("Creating req from pipfile: %s" % pipfile)
print("Using creation args: %s" % creation_args)
return cls(**creation_args)
@classmethod
@@ -669,7 +689,13 @@ class VCSRequirement(FileRequirement):
@property
def line_part(self):
"""requirements.txt compatible line part sans-extras"""
if self.req:
if self.is_local:
base_link = self.link
if not self.link:
base_link = self.get_link()
final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) if base_link.egg_fragment else "{0}"
base = final_format.format(self.vcs_uri)
elif self.req:
base = self.req.line
if base and self.extras and not extras_to_string(self.extras) in base:
if self.subdirectory:
@@ -694,7 +720,7 @@ class VCSRequirement(FileRequirement):
@property
def pipfile_part(self):
pipfile_dict = attr.asdict(self, filter=lambda k, v: v is not None and k.name != '_repo').copy()
pipfile_dict = attr.asdict(self, filter=lambda k, v: bool(v) is True and k.name != '_repo').copy()
if "vcs" in pipfile_dict:
pipfile_dict = self._choose_vcs_source(pipfile_dict)
name, _ = _strip_extras(pipfile_dict.pop("name"))
@@ -752,7 +778,10 @@ class Requirement(object):
def commit_hash(self):
if not self.is_vcs:
return None
return self.req.get_commit_hash()
commit_hash = None
with self.req.locked_vcs_repo() as repo:
commit_hash = repo.get_commit_hash()
return commit_hash
@specifiers.default
def get_specifiers(self):
pipenv/vendor/requirementslib/models/vcs.py
Vendored
+19 -10
View File
@@ -23,34 +23,43 @@ class VCSRepository(object):
backend = VCS_SUPPORT._registry.get(self.vcs_type)
return backend(url=self.url)
@property
def is_local(self):
url = self.url
if '+' in url:
url = url.split('+')[1]
return url.startswith("file")
def obtain(self):
if not os.path.exists(self.checkout_directory):
self.repo_instance.obtain(self.checkout_directory)
if self.ref:
with vistir.contextmanagers.cd(self.checkout_directory):
self.update(self.ref)
self.commit_sha = self.get_commit_hash()
self.checkout_ref(self.ref)
self.commit_sha = self.get_commit_hash(self.ref)
else:
if not self.commit_sha:
self.commit_sha = self.get_commit_hash()
def checkout_ref(self, ref):
target_rev = self.repo_instance.make_rev_options(ref)
if not self.repo_instance.is_commit_id_equal(
self.checkout_directory, self.get_commit_hash(ref)
) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref):
self.repo_instance.update(self.checkout_directory, self.url, target_rev)
self.commit_hash = self.get_commit_hash()
if not self.is_local:
self.update(ref)
def update(self, ref):
target_rev = self.repo_instance.make_rev_options(ref)
target_ref = self.repo_instance.make_rev_options(ref)
sha = self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev)
target_rev = target_ref.make_new(sha)
if parse_version(pip_version) > parse_version("18.0"):
self.repo_instance.update(self.checkout_directory, self.url, target_rev)
else:
self.repo_instance.update(self.checkout_directory, target_rev)
self.commit_hash = self.get_commit_hash()
self.repo_instance.update(self.checkout_directory, target_ref)
self.commit_hash = self.get_commit_hash(ref)
def get_commit_hash(self, ref=None):
if ref:
return self.repo_instance.get_revision(self.checkout_directory)
target_ref = self.repo_instance.make_rev_options(ref)
return self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev)
# return self.repo_instance.get_revision(self.checkout_directory)
return self.repo_instance.get_revision(self.checkout_directory)
tasks/vendoring/patches/patched/piptools.patch
+3 -3
View File
@@ -717,7 +717,7 @@ index 7e8cdf3..0a0d27d 100644
@@ -1,30 +1,42 @@
# -*- coding=utf-8 -*-
import importlib
-def do_import(module_path, subimport=None, old_path=None):
+
+def do_import(module_path, subimport=None, old_path=None, vendored_name=None):
@@ -744,8 +744,8 @@ index 7e8cdf3..0a0d27d 100644
if subimport:
return getattr(_tmp, subimport, _tmp)
return _tmp
-
-
-InstallRequirement = do_import('req.req_install', 'InstallRequirement')
-parse_requirements = do_import('req.req_file', 'parse_requirements')
-RequirementSet = do_import('req.req_set', 'RequirementSet')
tests/integration/test_install_uri.py
+5 -3
View File
@@ -97,16 +97,18 @@ def test_file_urls_work(PipenvInstance, pip_src_dir):
@pytest.mark.files
@pytest.mark.urls
@pytest.mark.needs_internet
def test_local_vcs_urls_work(PipenvInstance, pypi):
def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir):
six_dir = tmpdir.join("six")
six_path = Path(six_dir.strpath)
with PipenvInstance(pypi=pypi, chdir=True) as p:
six_path = Path(p.path).joinpath("six").absolute()
c = delegator.run(
"git clone " "https://github.com/benjaminp/six.git {0}".format(six_path)
"git clone https://github.com/benjaminp/six.git {0}".format(six_dir.strpath)
)
assert c.return_code == 0
c = p.pipenv("install git+{0}#egg=six".format(six_path.as_uri()))
assert c.return_code == 0
assert "six" in p.pipfile["packages"]
@pytest.mark.e