Merge pull request #3268 from pypa/bugfix/3255

Ignore hashes when using `--skip-lock`

news/3255.bugfix.rst

@@ -0,0 +1 @@
+Pipenv will now ignore hashes when installing with ``--skip-lock``.

pipenv/core.py

@@ -777,7 +777,8 @@ def do_install_dependencies(
             click.echo(
                 crayons.normal(fix_utf8("Installing dependencies from Pipfile…"), bold=True)
             )
-            lockfile = project.get_or_create_lockfile()
+            # skip_lock should completely bypass the lockfile (broken in 4dac1676)
+            lockfile = project.get_or_create_lockfile(from_pipfile=True)
     else:
         lockfile = project.get_or_create_lockfile()
         if not bare:
@@ -807,6 +808,8 @@ def do_install_dependencies(
 
     procs = queue.Queue(maxsize=PIPENV_MAX_SUBPROCESS)
     failed_deps_queue = queue.Queue()
+    if skip_lock:
+        ignore_hashes = True
 
     install_kwargs = {
         "no_deps": no_deps, "ignore_hashes": ignore_hashes, "allow_global": allow_global,

pipenv/project.py

@@ -743,10 +743,19 @@ class Project(object):
             source["verify_ssl"] = source["verify_ssl"].lower() == "true"
         return source
 
-    def get_or_create_lockfile(self):
+    def get_or_create_lockfile(self, from_pipfile=False):
         from pipenv.vendor.requirementslib.models.lockfile import Lockfile as Req_Lockfile
         lockfile = None
-        if self.lockfile_exists:
+        if from_pipfile and self.pipfile_exists:
+            lockfile_dict = {
+                "default": self._lockfile["default"].copy(),
+                "develop": self._lockfile["develop"].copy()
+            }
+            lockfile_dict.update({"_meta": self.get_lockfile_meta()})
+            lockfile = Req_Lockfile.from_data(
+                path=self.lockfile_location, data=lockfile_dict, meta_from_project=False
+            )
+        elif self.lockfile_exists:
             try:
                 lockfile = Req_Lockfile.load(self.lockfile_location)
             except OSError:
@@ -770,24 +779,16 @@ class Project(object):
             )
             lockfile._lockfile = lockfile.projectfile.model = _created_lockfile
             return lockfile
-        elif self.pipfile_exists:
-            lockfile_dict = {
-                "default": self._lockfile["default"].copy(),
-                "develop": self._lockfile["develop"].copy()
-            }
-            lockfile_dict.update({"_meta": self.get_lockfile_meta()})
-            _created_lockfile = Req_Lockfile.from_data(
-                path=self.lockfile_location, data=lockfile_dict, meta_from_project=False
-            )
-            lockfile._lockfile = _created_lockfile
-            return lockfile
+        else:
+            return self.get_or_create_lockfile(from_pipfile=True)
 
     def get_lockfile_meta(self):
         from .vendor.plette.lockfiles import PIPFILE_SPEC_CURRENT
-        sources = self.lockfile_content.get("_meta", {}).get("sources", [])
-        if not sources:
-            sources = self.pipfile_sources
-        elif not isinstance(sources, list):
+        if self.lockfile_exists:
+            sources = self.lockfile_content.get("_meta", {}).get("sources", [])
+        else:
+            sources = [dict(source) for source in self.parsed_pipfile["source"]]
+        if not isinstance(sources, list):
             sources = [sources,]
         return {
             "hash": {"sha256": self.calculate_pipfile_hash()},

pipenv/utils.py

@@ -320,7 +320,7 @@ class Resolver(object):
         if self.sources:
             requirementstxt_sources = " ".join(self.pip_args) if self.pip_args else ""
             requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--")
-        constraints_file.write(u"{0}\n".format(requirementstxt_sources))
+            constraints_file.write(u"{0}\n".format(requirementstxt_sources))
         constraints = self.initial_constraints
         constraints_file.write(u"\n".join([c for c in constraints]))
         constraints_file.close()
@@ -843,7 +843,6 @@ def mkdir_p(newdir):
                 if exn.errno != errno.EEXIST:
                     raise
 
-                
 
 def is_required_version(version, specified_version):
     """Check to see if there's a hard requirement for version

pipenv/vendor/plette/__init__.py

@@ -3,7 +3,7 @@ __all__ = [
     "Lockfile", "Pipfile",
 ]
 
-__version__ = '0.2.2'
+__version__ = '0.2.3.dev0'
 
 from .lockfiles import Lockfile
 from .pipfiles import Pipfile

pipenv/vendor/plette/models/base.py

@@ -22,7 +22,7 @@ def validate(cls, data):
         v = VALIDATORS[key]
     except KeyError:
         v = VALIDATORS[key] = cerberus.Validator(schema, allow_unknown=True)
-    if v.validate(data, normalize=False):
+    if v.validate(dict(data), normalize=False):
         return
     raise ValidationError(data, v)
 

pipenv/vendor/requirementslib/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding=utf-8 -*-
-__version__ = '1.3.1'
+__version__ = '1.3.2'
 
 import logging
 import warnings

pipenv/vendor/requirementslib/environment.py

@@ -0,0 +1,17 @@
+# -*- coding=utf-8 -*-
+from __future__ import print_function, absolute_import
+
+import os
+from appdirs import user_cache_dir
+
+
+def is_type_checking():
+    try:
+        from typing import TYPE_CHECKING
+    except ImportError:
+        return False
+    return TYPE_CHECKING
+
+
+REQUIREMENTSLIB_CACHE_DIR = os.getenv("REQUIREMENTSLIB_CACHE_DIR", user_cache_dir("pipenv"))
+MYPY_RUNNING = os.environ.get("MYPY_RUNNING", is_type_checking())

pipenv/vendor/requirementslib/models/pipfile.py

@@ -1,22 +1,81 @@
 # -*- coding: utf-8 -*-
 
-from __future__ import absolute_import, unicode_literals, print_function
+from __future__ import absolute_import, print_function, unicode_literals
 
-import attr
 import copy
 import os
+import sys
 
+import attr
 import tomlkit
 
-from vistir.compat import Path, FileNotFoundError
-
-from .requirements import Requirement
-from .project import ProjectFile
-from .utils import optional_instance_of
-from ..exceptions import RequirementError
-from ..utils import is_vcs, is_editable, merge_items
+import plette.models.base
 import plette.pipfiles
 
+from vistir.compat import FileNotFoundError, Path
+
+from ..exceptions import RequirementError
+from ..utils import is_editable, is_vcs, merge_items
+from .project import ProjectFile
+from .requirements import Requirement
+from .utils import optional_instance_of
+
+from ..environment import MYPY_RUNNING
+if MYPY_RUNNING:
+    from typing import Union, Any, Dict, Iterable, Sequence, Mapping, List, NoReturn
+    package_type = Dict[str, Dict[str, Union[List[str], str]]]
+    source_type = Dict[str, Union[str, bool]]
+    sources_type = Iterable[source_type]
+    meta_type = Dict[str, Union[int, Dict[str, str], sources_type]]
+    lockfile_type = Dict[str, Union[package_type, meta_type]]
+
+
+# Let's start by patching plette to make sure we can validate data without being broken
+try:
+    import cerberus
+except ImportError:
+    cerberus = None
+
+VALIDATORS = plette.models.base.VALIDATORS
+
+
+def patch_plette():
+    # type: () -> None
+
+    global VALIDATORS
+
+    def validate(cls, data):
+        # type: (Any, Dict[str, Any]) -> None
+        if not cerberus:    # Skip validation if Cerberus is not available.
+            return
+        schema = cls.__SCHEMA__
+        key = id(schema)
+        try:
+            v = VALIDATORS[key]
+        except KeyError:
+            v = VALIDATORS[key] = cerberus.Validator(schema, allow_unknown=True)
+        if v.validate(dict(data), normalize=False):
+            return
+        raise plette.models.base.ValidationError(data, v)
+
+    names = ["plette.models.base", plette.models.base.__name__]
+    names = [name for name in names if name in sys.modules]
+    for name in names:
+        if name in sys.modules:
+            module = sys.modules[name]
+        else:
+            module = plette.models.base
+        original_fn = getattr(module, "validate")
+        for key in ["__qualname__", "__name__", "__module__"]:
+            original_val = getattr(original_fn, key, None)
+            if original_val is not None:
+                setattr(validate, key, original_val)
+        setattr(module, "validate", validate)
+        sys.modules[name] = module
+
+
+patch_plette()
+
 
 is_pipfile = optional_instance_of(plette.pipfiles.Pipfile)
 is_path = optional_instance_of(Path)
@@ -24,8 +83,10 @@ is_projectfile = optional_instance_of(ProjectFile)
 
 
 def reorder_source_keys(data):
-    for i, entry in enumerate(data["source"]):
-        table = tomlkit.table()
+    # type: ignore
+    sources = data["source"]  # type: sources_type
+    for i, entry in enumerate(sources):
+        table = tomlkit.table()  # type: Mapping
         table["name"] = entry["name"]
         table["url"] = entry["url"]
         table["verify_ssl"] = entry["verify_ssl"]
@@ -36,6 +97,7 @@ def reorder_source_keys(data):
 class PipfileLoader(plette.pipfiles.Pipfile):
     @classmethod
     def validate(cls, data):
+        # type: (Dict[str, Any]) -> None
         for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items():
             if key not in data or key == "source":
                 continue
@@ -46,6 +108,7 @@ class PipfileLoader(plette.pipfiles.Pipfile):
 
     @classmethod
     def load(cls, f, encoding=None):
+        # type: (Any, str) -> PipfileLoader
         content = f.read()
         if encoding is not None:
             content = content.decode(encoding)
@@ -69,6 +132,7 @@ class PipfileLoader(plette.pipfiles.Pipfile):
         return instance
 
     def __getattribute__(self, key):
+        # type: (str) -> Any
         if key == "source":
             return self._data[key]
         return super(PipfileLoader, self).__getattribute__(key)
@@ -78,7 +142,7 @@ class PipfileLoader(plette.pipfiles.Pipfile):
 class Pipfile(object):
     path = attr.ib(validator=is_path, type=Path)
     projectfile = attr.ib(validator=is_projectfile, type=ProjectFile)
-    _pipfile = attr.ib(type=plette.pipfiles.Pipfile)
+    _pipfile = attr.ib(type=PipfileLoader)
     _pyproject = attr.ib(default=attr.Factory(tomlkit.document), type=tomlkit.toml_document.TOMLDocument)
     build_system = attr.ib(default=attr.Factory(dict), type=dict)
     requirements = attr.ib(default=attr.Factory(list), type=list)
@@ -86,22 +150,27 @@ class Pipfile(object):
 
     @path.default
     def _get_path(self):
+        # type: () -> Path
         return Path(os.curdir).absolute()
 
     @projectfile.default
     def _get_projectfile(self):
+        # type: () -> ProjectFile
         return self.load_projectfile(os.curdir, create=False)
 
     @_pipfile.default
     def _get_pipfile(self):
+        # type: () -> Union[plette.pipfiles.Pipfile, PipfileLoader]
         return self.projectfile.model
 
     @property
     def pipfile(self):
+        # type: () -> Union[PipfileLoader, plette.pipfiles.Pipfile]
         return self._pipfile
 
     def get_deps(self, dev=False, only=True):
-        deps = {}
+        # type: (bool, bool) -> Dict[str, Dict[str, Union[List[str], str]]]
+        deps = {}  # type: Dict[str, Dict[str, Union[List[str], str]]]
         if dev:
             deps.update(self.pipfile._data["dev-packages"])
             if only:
@@ -109,15 +178,18 @@ class Pipfile(object):
         return merge_items([deps, self.pipfile._data["packages"]])
 
     def get(self, k):
+        # type: (str) -> Any
         return self.__getitem__(k)
 
     def __contains__(self, k):
+        # type: (str) -> bool
         check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k)
         if check_pipfile:
             return True
-        return super(Pipfile, self).__contains__(k)
+        return False
 
     def __getitem__(self, k, *args, **kwargs):
+        # type: ignore
         retval = None
         pipfile = self._pipfile
         section = None
@@ -139,6 +211,7 @@ class Pipfile(object):
         return retval
 
     def __getattr__(self, k, *args, **kwargs):
+        # type: ignore
         retval = None
         pipfile = super(Pipfile, self).__getattribute__("_pipfile")
         try:
@@ -151,14 +224,17 @@ class Pipfile(object):
 
     @property
     def requires_python(self):
+        # type: () -> bool
         return self._pipfile.requires.requires_python
 
     @property
     def allow_prereleases(self):
+        # type: () -> bool
         return self._pipfile.get("pipenv", {}).get("allow_prereleases", False)
 
     @classmethod
     def read_projectfile(cls, path):
+        # type: (str) -> ProjectFile
         """Read the specified project file and provide an interface for writing/updating.
 
         :param str path: Path to the target file.
@@ -174,6 +250,7 @@ class Pipfile(object):
 
     @classmethod
     def load_projectfile(cls, path, create=False):
+        # type: (str, bool) -> ProjectFile
         """Given a path, load or create the necessary pipfile.
 
         :param str path: Path to the project root or pipfile
@@ -198,6 +275,7 @@ class Pipfile(object):
 
     @classmethod
     def load(cls, path, create=False):
+        # type: (str, bool) -> Pipfile
         """Given a path, load or create the necessary pipfile.
 
         :param str path: Path to the project root or pipfile
@@ -226,22 +304,22 @@ class Pipfile(object):
         return cls(**creation_args)
 
     def write(self):
+        # type: () -> None
         self.projectfile.model = copy.deepcopy(self._pipfile)
         self.projectfile.write()
 
     @property
-    def dev_packages(self, as_requirements=True):
-        if as_requirements:
-            return self.dev_requirements
-        return self._pipfile.get('dev-packages', {})
+    def dev_packages(self):
+        # type: () -> List[Requirement]
+        return self.dev_requirements
 
     @property
-    def packages(self, as_requirements=True):
-        if as_requirements:
-            return self.requirements
-        return self._pipfile.get('packages', {})
+    def packages(self):
+        # type: () -> List[Requirement]
+        return self.requirements
 
     def _read_pyproject(self):
+        # type: () -> None
         pyproject = self.path.parent.joinpath("pyproject.toml")
         if pyproject.exists():
             self._pyproject = tomlkit.load(pyproject)
@@ -256,8 +334,10 @@ class Pipfile(object):
 
     @property
     def build_requires(self):
+        # type: () -> List[str]
         return self.build_system.get("requires", [])
 
     @property
     def build_backend(self):
+        # type: () -> str
         return self.build_system.get("build-backend", None)

pipenv/vendor/requirementslib/models/requirements.py

@@ -3,6 +3,7 @@
 from __future__ import absolute_import
 
 import collections
+import copy
 import hashlib
 import os
 
@@ -312,9 +313,19 @@ class FileRequirement(object):
         )):
             if self.editable:
                 line = pip_shims.shims.path_to_url(self.setup_py_dir)
+                if self.extras:
+                    line = "{0}[{1}]".format(line, ",".join(self.extras))
                 _ireq = pip_shims.shims.install_req_from_editable(line)
             else:
-                _ireq = pip_shims.shims.install_req_from_line(Path(self.setup_py_dir).as_posix())
+                line = Path(self.setup_py_dir).as_posix()
+                if self.extras:
+                    line = "{0}[{1}]".format(line, ",".join(self.extras))
+                _ireq = pip_shims.shims.install_req_from_line(line)
+            if self.req:
+                _ireq.req = copy.deepcopy(self.req)
+            else:
+                if self.extras:
+                    _ireq.extras = set(self.extras)
             from .setup_info import SetupInfo
             subdir = getattr(self, "subdirectory", None)
             setupinfo = SetupInfo.from_ireq(_ireq, subdir=subdir)
@@ -453,10 +464,16 @@ class FileRequirement(object):
         if not name:
             _line = unquote(link.url_without_fragment) if link.url else uri
             if editable:
+                if extras:
+                    _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras))))
                 ireq = pip_shims.shims.install_req_from_editable(_line)
             else:
                 _line = path if (uri_scheme and uri_scheme == "path") else _line
+                if extras:
+                    _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras))))
                 ireq = pip_shims.shims.install_req_from_line(_line)
+            if extras and not ireq.extras:
+                ireq.extras = set(extras)
             setup_info = SetupInfo.from_ireq(ireq)
             setupinfo_dict = setup_info.as_dict()
             setup_name = setupinfo_dict.get("name", None)
@@ -488,7 +505,7 @@ class FileRequirement(object):
         return cls_inst
 
     @classmethod
-    def from_line(cls, line):
+    def from_line(cls, line, extras=None):
         line = line.strip('"').strip("'")
         link = None
         path = None
@@ -497,6 +514,8 @@ class FileRequirement(object):
         setup_path = None
         name = None
         req = None
+        if not extras:
+            extras = []
         if not any([is_installable_file(line), is_valid_url(line), is_file_url(line)]):
             try:
                 req = init_requirement(line)
@@ -515,7 +534,8 @@ class FileRequirement(object):
             "editable": editable,
             "setup_path": setup_path,
             "uri_scheme": prefer,
-            "line": line
+            "line": line,
+            "extras": extras
         }
         if link and link.is_wheel:
             from pip_shims import Wheel
@@ -1069,7 +1089,7 @@ class Requirement(object):
             (is_valid_url(possible_url) or is_file_url(line) or is_valid_url(line)) and
             not (line_is_vcs or is_vcs(possible_url))
         ):
-            r = FileRequirement.from_line(line_with_prefix)
+            r = FileRequirement.from_line(line_with_prefix, extras=extras)
         elif line_is_vcs:
             r = VCSRequirement.from_line(line_with_prefix, extras=extras)
             vcs = r.vcs

pipenv/vendor/requirementslib/models/setup_info.py

@@ -7,6 +7,7 @@ import attr
 import packaging.version
 import packaging.specifiers
 import packaging.utils
+import six
 
 try:
     from setuptools.dist import distutils
@@ -67,6 +68,18 @@ def _get_src_dir():
     return os.path.join(os.getcwd(), "src")  # Match pip's behavior.
 
 
+def ensure_reqs(reqs):
+    import pkg_resources
+    new_reqs = []
+    for req in reqs:
+        if not req:
+            continue
+        if isinstance(req, six.string_types):
+            req = pkg_resources.Requirement.parse("{0}".format(str(req)))
+        new_reqs.append(req)
+    return new_reqs
+
+
 def _prepare_wheel_building_kwargs(ireq):
     download_dir = os.path.join(CACHE_DIR, "pkgs")
     mkdir_p(download_dir)
@@ -153,10 +166,8 @@ def get_metadata(path, pkg_name=None):
                     else:
                         marker = ""
                         extra = "{0}".format(k)
-                    _deps = [
-                        pkg_resources.Requirement.parse("{0}{1}".format(str(req), marker))
-                        for req in _deps
-                    ]
+                    _deps = ["{0}{1}".format(str(req), marker) for req in _deps]
+                    _deps = ensure_reqs(_deps)
                     if extra:
                         extras[extra] = _deps
                     else:
@@ -220,19 +231,25 @@ class SetupInfo(object):
                 python_requires = parser.get("options", "python_requires")
                 if python_requires and not self.python_requires:
                     self.python_requires = python_requires
-            if parser.has_option("options", "extras_require"):
+            if "options.extras_require" in parser.sections():
                 self.extras.update(
                     {
                         section: [
-                            dep.strip()
+                            init_requirement(dep.strip())
                             for dep in parser.get(
                                 "options.extras_require", section
                             ).split("\n")
                             if dep
                         ]
                         for section in parser.options("options.extras_require")
+                        if section not in ["options", "metadata"]
                     }
                 )
+                if self.ireq.extras:
+                    self.requires.update({
+                        extra: self.extras[extra]
+                        for extra in self.ireq.extras if extra in self.extras
+                    })
 
     def run_setup(self):
         if self.setup_py is not None and self.setup_py.exists():
@@ -304,12 +321,11 @@ class SetupInfo(object):
                 )
                 if getattr(self.ireq, "extras", None):
                     for extra in self.ireq.extras:
+                        extras = metadata.get("extras", {}).get(extra)
+                        extras = ensure_reqs(extras)
+                        self.extras[extra] = set(extras)
                         self.requires.update(
-                            {
-                                req.key: req for req
-                                in metadata.get("extras", {}).get(extra)
-                                if req is not None
-                            }
+                            {req.key: req for req in extras if req is not None}
                         )
 
     def run_pyproject(self):

pipenv/vendor/vendor.txt

@@ -32,7 +32,7 @@ requirementslib==1.3.1.post1
     distlib==0.2.8
     packaging==18.0
     pyparsing==2.2.2
-    plette==0.2.2
+    git+https://github.com/sarugaku/plette.git@master#egg=plette
         tomlkit==0.5.2
 shellingham==1.2.7
 six==1.11.0