diff --git a/news/3255.bugfix.rst b/news/3255.bugfix.rst new file mode 100644 index 00000000..72c6aaf7 --- /dev/null +++ b/news/3255.bugfix.rst @@ -0,0 +1 @@ +Pipenv will now ignore hashes when installing with ``--skip-lock``. diff --git a/pipenv/core.py b/pipenv/core.py index 6f1a7a2d..7f488aea 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -777,7 +777,8 @@ def do_install_dependencies( click.echo( crayons.normal(fix_utf8("Installing dependencies from Pipfile…"), bold=True) ) - lockfile = project.get_or_create_lockfile() + # skip_lock should completely bypass the lockfile (broken in 4dac1676) + lockfile = project.get_or_create_lockfile(from_pipfile=True) else: lockfile = project.get_or_create_lockfile() if not bare: @@ -807,6 +808,8 @@ def do_install_dependencies( procs = queue.Queue(maxsize=PIPENV_MAX_SUBPROCESS) failed_deps_queue = queue.Queue() + if skip_lock: + ignore_hashes = True install_kwargs = { "no_deps": no_deps, "ignore_hashes": ignore_hashes, "allow_global": allow_global, diff --git a/pipenv/project.py b/pipenv/project.py index 69f3ad5d..ef34eeb8 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -743,10 +743,19 @@ class Project(object): source["verify_ssl"] = source["verify_ssl"].lower() == "true" return source - def get_or_create_lockfile(self): + def get_or_create_lockfile(self, from_pipfile=False): from pipenv.vendor.requirementslib.models.lockfile import Lockfile as Req_Lockfile lockfile = None - if self.lockfile_exists: + if from_pipfile and self.pipfile_exists: + lockfile_dict = { + "default": self._lockfile["default"].copy(), + "develop": self._lockfile["develop"].copy() + } + lockfile_dict.update({"_meta": self.get_lockfile_meta()}) + lockfile = Req_Lockfile.from_data( + path=self.lockfile_location, data=lockfile_dict, meta_from_project=False + ) + elif self.lockfile_exists: try: lockfile = Req_Lockfile.load(self.lockfile_location) except OSError: @@ -770,24 +779,16 @@ class Project(object): ) lockfile._lockfile = lockfile.projectfile.model = _created_lockfile return lockfile - elif self.pipfile_exists: - lockfile_dict = { - "default": self._lockfile["default"].copy(), - "develop": self._lockfile["develop"].copy() - } - lockfile_dict.update({"_meta": self.get_lockfile_meta()}) - _created_lockfile = Req_Lockfile.from_data( - path=self.lockfile_location, data=lockfile_dict, meta_from_project=False - ) - lockfile._lockfile = _created_lockfile - return lockfile + else: + return self.get_or_create_lockfile(from_pipfile=True) def get_lockfile_meta(self): from .vendor.plette.lockfiles import PIPFILE_SPEC_CURRENT - sources = self.lockfile_content.get("_meta", {}).get("sources", []) - if not sources: - sources = self.pipfile_sources - elif not isinstance(sources, list): + if self.lockfile_exists: + sources = self.lockfile_content.get("_meta", {}).get("sources", []) + else: + sources = [dict(source) for source in self.parsed_pipfile["source"]] + if not isinstance(sources, list): sources = [sources,] return { "hash": {"sha256": self.calculate_pipfile_hash()}, diff --git a/pipenv/utils.py b/pipenv/utils.py index 84a4c105..5ee8158e 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -320,7 +320,7 @@ class Resolver(object): if self.sources: requirementstxt_sources = " ".join(self.pip_args) if self.pip_args else "" requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") - constraints_file.write(u"{0}\n".format(requirementstxt_sources)) + constraints_file.write(u"{0}\n".format(requirementstxt_sources)) constraints = self.initial_constraints constraints_file.write(u"\n".join([c for c in constraints])) constraints_file.close() @@ -843,7 +843,6 @@ def mkdir_p(newdir): if exn.errno != errno.EEXIST: raise - def is_required_version(version, specified_version): """Check to see if there's a hard requirement for version diff --git a/pipenv/vendor/plette/__init__.py b/pipenv/vendor/plette/__init__.py index 8099f0b1..5daf460c 100644 --- a/pipenv/vendor/plette/__init__.py +++ b/pipenv/vendor/plette/__init__.py @@ -3,7 +3,7 @@ __all__ = [ "Lockfile", "Pipfile", ] -__version__ = '0.2.2' +__version__ = '0.2.3.dev0' from .lockfiles import Lockfile from .pipfiles import Pipfile diff --git a/pipenv/vendor/plette/models/base.py b/pipenv/vendor/plette/models/base.py index d70752ee..fad0d09e 100644 --- a/pipenv/vendor/plette/models/base.py +++ b/pipenv/vendor/plette/models/base.py @@ -22,7 +22,7 @@ def validate(cls, data): v = VALIDATORS[key] except KeyError: v = VALIDATORS[key] = cerberus.Validator(schema, allow_unknown=True) - if v.validate(data, normalize=False): + if v.validate(dict(data), normalize=False): return raise ValidationError(data, v) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 32415c61..aca59917 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = '1.3.1' +__version__ = '1.3.2' import logging import warnings diff --git a/pipenv/vendor/requirementslib/environment.py b/pipenv/vendor/requirementslib/environment.py new file mode 100644 index 00000000..2a7d9b0e --- /dev/null +++ b/pipenv/vendor/requirementslib/environment.py @@ -0,0 +1,17 @@ +# -*- coding=utf-8 -*- +from __future__ import print_function, absolute_import + +import os +from appdirs import user_cache_dir + + +def is_type_checking(): + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +REQUIREMENTSLIB_CACHE_DIR = os.getenv("REQUIREMENTSLIB_CACHE_DIR", user_cache_dir("pipenv")) +MYPY_RUNNING = os.environ.get("MYPY_RUNNING", is_type_checking()) diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index e3d353d9..84a4a26d 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -1,22 +1,81 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import absolute_import, print_function, unicode_literals -import attr import copy import os +import sys +import attr import tomlkit -from vistir.compat import Path, FileNotFoundError - -from .requirements import Requirement -from .project import ProjectFile -from .utils import optional_instance_of -from ..exceptions import RequirementError -from ..utils import is_vcs, is_editable, merge_items +import plette.models.base import plette.pipfiles +from vistir.compat import FileNotFoundError, Path + +from ..exceptions import RequirementError +from ..utils import is_editable, is_vcs, merge_items +from .project import ProjectFile +from .requirements import Requirement +from .utils import optional_instance_of + +from ..environment import MYPY_RUNNING +if MYPY_RUNNING: + from typing import Union, Any, Dict, Iterable, Sequence, Mapping, List, NoReturn + package_type = Dict[str, Dict[str, Union[List[str], str]]] + source_type = Dict[str, Union[str, bool]] + sources_type = Iterable[source_type] + meta_type = Dict[str, Union[int, Dict[str, str], sources_type]] + lockfile_type = Dict[str, Union[package_type, meta_type]] + + +# Let's start by patching plette to make sure we can validate data without being broken +try: + import cerberus +except ImportError: + cerberus = None + +VALIDATORS = plette.models.base.VALIDATORS + + +def patch_plette(): + # type: () -> None + + global VALIDATORS + + def validate(cls, data): + # type: (Any, Dict[str, Any]) -> None + if not cerberus: # Skip validation if Cerberus is not available. + return + schema = cls.__SCHEMA__ + key = id(schema) + try: + v = VALIDATORS[key] + except KeyError: + v = VALIDATORS[key] = cerberus.Validator(schema, allow_unknown=True) + if v.validate(dict(data), normalize=False): + return + raise plette.models.base.ValidationError(data, v) + + names = ["plette.models.base", plette.models.base.__name__] + names = [name for name in names if name in sys.modules] + for name in names: + if name in sys.modules: + module = sys.modules[name] + else: + module = plette.models.base + original_fn = getattr(module, "validate") + for key in ["__qualname__", "__name__", "__module__"]: + original_val = getattr(original_fn, key, None) + if original_val is not None: + setattr(validate, key, original_val) + setattr(module, "validate", validate) + sys.modules[name] = module + + +patch_plette() + is_pipfile = optional_instance_of(plette.pipfiles.Pipfile) is_path = optional_instance_of(Path) @@ -24,8 +83,10 @@ is_projectfile = optional_instance_of(ProjectFile) def reorder_source_keys(data): - for i, entry in enumerate(data["source"]): - table = tomlkit.table() + # type: ignore + sources = data["source"] # type: sources_type + for i, entry in enumerate(sources): + table = tomlkit.table() # type: Mapping table["name"] = entry["name"] table["url"] = entry["url"] table["verify_ssl"] = entry["verify_ssl"] @@ -36,6 +97,7 @@ def reorder_source_keys(data): class PipfileLoader(plette.pipfiles.Pipfile): @classmethod def validate(cls, data): + # type: (Dict[str, Any]) -> None for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items(): if key not in data or key == "source": continue @@ -46,6 +108,7 @@ class PipfileLoader(plette.pipfiles.Pipfile): @classmethod def load(cls, f, encoding=None): + # type: (Any, str) -> PipfileLoader content = f.read() if encoding is not None: content = content.decode(encoding) @@ -69,6 +132,7 @@ class PipfileLoader(plette.pipfiles.Pipfile): return instance def __getattribute__(self, key): + # type: (str) -> Any if key == "source": return self._data[key] return super(PipfileLoader, self).__getattribute__(key) @@ -78,7 +142,7 @@ class PipfileLoader(plette.pipfiles.Pipfile): class Pipfile(object): path = attr.ib(validator=is_path, type=Path) projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) - _pipfile = attr.ib(type=plette.pipfiles.Pipfile) + _pipfile = attr.ib(type=PipfileLoader) _pyproject = attr.ib(default=attr.Factory(tomlkit.document), type=tomlkit.toml_document.TOMLDocument) build_system = attr.ib(default=attr.Factory(dict), type=dict) requirements = attr.ib(default=attr.Factory(list), type=list) @@ -86,22 +150,27 @@ class Pipfile(object): @path.default def _get_path(self): + # type: () -> Path return Path(os.curdir).absolute() @projectfile.default def _get_projectfile(self): + # type: () -> ProjectFile return self.load_projectfile(os.curdir, create=False) @_pipfile.default def _get_pipfile(self): + # type: () -> Union[plette.pipfiles.Pipfile, PipfileLoader] return self.projectfile.model @property def pipfile(self): + # type: () -> Union[PipfileLoader, plette.pipfiles.Pipfile] return self._pipfile def get_deps(self, dev=False, only=True): - deps = {} + # type: (bool, bool) -> Dict[str, Dict[str, Union[List[str], str]]] + deps = {} # type: Dict[str, Dict[str, Union[List[str], str]]] if dev: deps.update(self.pipfile._data["dev-packages"]) if only: @@ -109,15 +178,18 @@ class Pipfile(object): return merge_items([deps, self.pipfile._data["packages"]]) def get(self, k): + # type: (str) -> Any return self.__getitem__(k) def __contains__(self, k): + # type: (str) -> bool check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) if check_pipfile: return True - return super(Pipfile, self).__contains__(k) + return False def __getitem__(self, k, *args, **kwargs): + # type: ignore retval = None pipfile = self._pipfile section = None @@ -139,6 +211,7 @@ class Pipfile(object): return retval def __getattr__(self, k, *args, **kwargs): + # type: ignore retval = None pipfile = super(Pipfile, self).__getattribute__("_pipfile") try: @@ -151,14 +224,17 @@ class Pipfile(object): @property def requires_python(self): + # type: () -> bool return self._pipfile.requires.requires_python @property def allow_prereleases(self): + # type: () -> bool return self._pipfile.get("pipenv", {}).get("allow_prereleases", False) @classmethod def read_projectfile(cls, path): + # type: (str) -> ProjectFile """Read the specified project file and provide an interface for writing/updating. :param str path: Path to the target file. @@ -174,6 +250,7 @@ class Pipfile(object): @classmethod def load_projectfile(cls, path, create=False): + # type: (str, bool) -> ProjectFile """Given a path, load or create the necessary pipfile. :param str path: Path to the project root or pipfile @@ -198,6 +275,7 @@ class Pipfile(object): @classmethod def load(cls, path, create=False): + # type: (str, bool) -> Pipfile """Given a path, load or create the necessary pipfile. :param str path: Path to the project root or pipfile @@ -226,22 +304,22 @@ class Pipfile(object): return cls(**creation_args) def write(self): + # type: () -> None self.projectfile.model = copy.deepcopy(self._pipfile) self.projectfile.write() @property - def dev_packages(self, as_requirements=True): - if as_requirements: - return self.dev_requirements - return self._pipfile.get('dev-packages', {}) + def dev_packages(self): + # type: () -> List[Requirement] + return self.dev_requirements @property - def packages(self, as_requirements=True): - if as_requirements: - return self.requirements - return self._pipfile.get('packages', {}) + def packages(self): + # type: () -> List[Requirement] + return self.requirements def _read_pyproject(self): + # type: () -> None pyproject = self.path.parent.joinpath("pyproject.toml") if pyproject.exists(): self._pyproject = tomlkit.load(pyproject) @@ -256,8 +334,10 @@ class Pipfile(object): @property def build_requires(self): + # type: () -> List[str] return self.build_system.get("requires", []) @property def build_backend(self): + # type: () -> str return self.build_system.get("build-backend", None) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index d034a12d..bfd69b37 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -3,6 +3,7 @@ from __future__ import absolute_import import collections +import copy import hashlib import os @@ -312,9 +313,19 @@ class FileRequirement(object): )): if self.editable: line = pip_shims.shims.path_to_url(self.setup_py_dir) + if self.extras: + line = "{0}[{1}]".format(line, ",".join(self.extras)) _ireq = pip_shims.shims.install_req_from_editable(line) else: - _ireq = pip_shims.shims.install_req_from_line(Path(self.setup_py_dir).as_posix()) + line = Path(self.setup_py_dir).as_posix() + if self.extras: + line = "{0}[{1}]".format(line, ",".join(self.extras)) + _ireq = pip_shims.shims.install_req_from_line(line) + if self.req: + _ireq.req = copy.deepcopy(self.req) + else: + if self.extras: + _ireq.extras = set(self.extras) from .setup_info import SetupInfo subdir = getattr(self, "subdirectory", None) setupinfo = SetupInfo.from_ireq(_ireq, subdir=subdir) @@ -453,10 +464,16 @@ class FileRequirement(object): if not name: _line = unquote(link.url_without_fragment) if link.url else uri if editable: + if extras: + _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) ireq = pip_shims.shims.install_req_from_editable(_line) else: _line = path if (uri_scheme and uri_scheme == "path") else _line + if extras: + _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) ireq = pip_shims.shims.install_req_from_line(_line) + if extras and not ireq.extras: + ireq.extras = set(extras) setup_info = SetupInfo.from_ireq(ireq) setupinfo_dict = setup_info.as_dict() setup_name = setupinfo_dict.get("name", None) @@ -488,7 +505,7 @@ class FileRequirement(object): return cls_inst @classmethod - def from_line(cls, line): + def from_line(cls, line, extras=None): line = line.strip('"').strip("'") link = None path = None @@ -497,6 +514,8 @@ class FileRequirement(object): setup_path = None name = None req = None + if not extras: + extras = [] if not any([is_installable_file(line), is_valid_url(line), is_file_url(line)]): try: req = init_requirement(line) @@ -515,7 +534,8 @@ class FileRequirement(object): "editable": editable, "setup_path": setup_path, "uri_scheme": prefer, - "line": line + "line": line, + "extras": extras } if link and link.is_wheel: from pip_shims import Wheel @@ -1069,7 +1089,7 @@ class Requirement(object): (is_valid_url(possible_url) or is_file_url(line) or is_valid_url(line)) and not (line_is_vcs or is_vcs(possible_url)) ): - r = FileRequirement.from_line(line_with_prefix) + r = FileRequirement.from_line(line_with_prefix, extras=extras) elif line_is_vcs: r = VCSRequirement.from_line(line_with_prefix, extras=extras) vcs = r.vcs diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index c32c2790..076ef7ed 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -7,6 +7,7 @@ import attr import packaging.version import packaging.specifiers import packaging.utils +import six try: from setuptools.dist import distutils @@ -67,6 +68,18 @@ def _get_src_dir(): return os.path.join(os.getcwd(), "src") # Match pip's behavior. +def ensure_reqs(reqs): + import pkg_resources + new_reqs = [] + for req in reqs: + if not req: + continue + if isinstance(req, six.string_types): + req = pkg_resources.Requirement.parse("{0}".format(str(req))) + new_reqs.append(req) + return new_reqs + + def _prepare_wheel_building_kwargs(ireq): download_dir = os.path.join(CACHE_DIR, "pkgs") mkdir_p(download_dir) @@ -153,10 +166,8 @@ def get_metadata(path, pkg_name=None): else: marker = "" extra = "{0}".format(k) - _deps = [ - pkg_resources.Requirement.parse("{0}{1}".format(str(req), marker)) - for req in _deps - ] + _deps = ["{0}{1}".format(str(req), marker) for req in _deps] + _deps = ensure_reqs(_deps) if extra: extras[extra] = _deps else: @@ -220,19 +231,25 @@ class SetupInfo(object): python_requires = parser.get("options", "python_requires") if python_requires and not self.python_requires: self.python_requires = python_requires - if parser.has_option("options", "extras_require"): + if "options.extras_require" in parser.sections(): self.extras.update( { section: [ - dep.strip() + init_requirement(dep.strip()) for dep in parser.get( "options.extras_require", section ).split("\n") if dep ] for section in parser.options("options.extras_require") + if section not in ["options", "metadata"] } ) + if self.ireq.extras: + self.requires.update({ + extra: self.extras[extra] + for extra in self.ireq.extras if extra in self.extras + }) def run_setup(self): if self.setup_py is not None and self.setup_py.exists(): @@ -304,12 +321,11 @@ class SetupInfo(object): ) if getattr(self.ireq, "extras", None): for extra in self.ireq.extras: + extras = metadata.get("extras", {}).get(extra) + extras = ensure_reqs(extras) + self.extras[extra] = set(extras) self.requires.update( - { - req.key: req for req - in metadata.get("extras", {}).get(extra) - if req is not None - } + {req.key: req for req in extras if req is not None} ) def run_pyproject(self): diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index ff7226b2..456ce457 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -32,7 +32,7 @@ requirementslib==1.3.1.post1 distlib==0.2.8 packaging==18.0 pyparsing==2.2.2 - plette==0.2.2 + git+https://github.com/sarugaku/plette.git@master#egg=plette tomlkit==0.5.2 shellingham==1.2.7 six==1.11.0