diff --git a/README.md b/README.md index 74067d58..ea2bf074 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,7 @@ Pipenv: Python Development Workflow for Humans [![image](https://img.shields.io/pypi/v/pipenv.svg)](https://python.org/pypi/pipenv) [![image](https://img.shields.io/pypi/l/pipenv.svg)](https://python.org/pypi/pipenv) -[![image](https://badge.buildkite.com/79c7eccf056b17c3151f3c4d0e4c4b8b724539d84f1e037b9b.svg?branch=master)](https://code.kennethreitz.org/source/pipenv/) -[![Azure Pipelines build status (Linux)](https://dev.azure.com/pypa/pipenv/_apis/build/status/pipenv%20CI%20(Linux)?branchName=master&label=Linux)](https://dev.azure.com/pypa/pipenv/_build/latest?definitionId=13&branchName=master) -[![Azure Pipelines build status (Windows)](https://dev.azure.com/pypa/pipenv/_apis/build/status/pipenv%20CI%20(Windows)?branchName=master&label=Windows)](https://dev.azure.com/pypa/pipenv/_build/latest?definitionId=12&branchName=master) +[![Azure Pipelines Build Status](https://dev.azure.com/pypa/pipenv/_apis/build/status/Pipenv%20CI?branchName=master)](https://dev.azure.com/pypa/pipenv/_build/latest?definitionId=16&branchName=master) [![image](https://img.shields.io/pypi/pyversions/pipenv.svg)](https://python.org/pypi/pipenv) [![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz) diff --git a/news/3766.bugfix.rst b/news/3766.bugfix.rst new file mode 100644 index 00000000..f7f7d304 --- /dev/null +++ b/news/3766.bugfix.rst @@ -0,0 +1 @@ +``pipenv update --outdated`` will now correctly handle comparisons between pre/post-releases and normal releases. diff --git a/news/3766.vendor.rst b/news/3766.vendor.rst new file mode 100644 index 00000000..16ebbed9 --- /dev/null +++ b/news/3766.vendor.rst @@ -0,0 +1 @@ +Updated ``pip_shims`` to support ``--outdated`` with new pip versions. diff --git a/pipenv/core.py b/pipenv/core.py index 32f5e2fa..8490cdf2 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -26,7 +26,7 @@ from .environments import ( PIPENV_CACHE_DIR, PIPENV_COLORBLIND, PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_DONT_USE_PYENV, PIPENV_HIDE_EMOJIS, PIPENV_MAX_SUBPROCESS, PIPENV_PYUP_API_KEY, PIPENV_SHELL_FANCY, PIPENV_SKIP_VALIDATION, - PIPENV_YES, SESSION_IS_INTERACTIVE, PIP_EXISTS_ACTION + PIPENV_YES, SESSION_IS_INTERACTIVE, PIP_EXISTS_ACTION, PIPENV_RESOLVE_VCS ) from .project import Project, SourceNotFound from .utils import ( @@ -1798,7 +1798,9 @@ def do_py(system=False): def do_outdated(pypi_mirror=None): # TODO: Allow --skip-lock here? from .vendor.requirementslib.models.requirements import Requirement + from .vendor.requirementslib.models.utils import get_version from .vendor.packaging.utils import canonicalize_name + from .vendor.vistir.compat import Mapping from collections import namedtuple packages = {} @@ -1810,6 +1812,7 @@ def do_outdated(pypi_mirror=None): (pkg.project_name, pkg.parsed_version, pkg.latest_version) for pkg in project.environment.get_outdated_packages() } + reverse_deps = project.environment.reverse_dependencies() for result in installed_packages: dep = Requirement.from_line(str(result.as_requirement())) packages.update(dep.as_pipfile()) @@ -1833,10 +1836,26 @@ def do_outdated(pypi_mirror=None): elif canonicalize_name(package) in outdated_packages: skipped.append(outdated_packages[canonicalize_name(package)]) for package, old_version, new_version in skipped: - click.echo(crayons.yellow( - "Skipped Update of Package {0!s}: {1!s} installed, {2!s} available.".format( - package, old_version, new_version - )), err=True + name_in_pipfile = project.get_package_name_in_pipfile(package) + pipfile_version_text = "" + required = "" + version = None + if name_in_pipfile: + version = get_version(project.packages[name_in_pipfile]) + reverse_deps = reverse_deps.get(name_in_pipfile) + if isinstance(reverse_deps, Mapping) and "required" in reverse_deps: + required = " {0} required".format(reverse_deps["required"]) + if version: + pipfile_version_text = " ({0} set in Pipfile)".format(version) + else: + pipfile_version_text = " (Unpinned in Pipfile)" + click.echo( + crayons.yellow( + "Skipped Update of Package {0!s}: {1!s} installed,{2!s}{3!s}, " + "{4!s} available.".format( + package, old_version, required, pipfile_version_text, new_version + ) + ), err=True ) if not outdated: click.echo(crayons.green("All packages are up to date!", bold=True)) @@ -2064,6 +2083,7 @@ def do_install( os.environ["PIP_USER"] = vistir.compat.fs_str("0") if "PYTHONHOME" in os.environ: del os.environ["PYTHONHOME"] + sp.text = "Resolving {0}...".format(pkg_line) try: pkg_requirement = Requirement.from_line(pkg_line) except ValueError as e: @@ -2072,30 +2092,45 @@ def do_install( sys.exit(1) if index_url: pkg_requirement.index = index_url + deps = [] + if pkg_requirement.is_vcs and PIPENV_RESOLVE_VCS: + deps = pkg_requirement.req.dependencies + to_install = [pkg_requirement,] + no_deps = False + sp.text = "Installing..." try: - c = pip_install( - pkg_requirement, - ignore_hashes=True, - allow_global=system, - selective_upgrade=selective_upgrade, - no_deps=False, - pre=pre, - requirements_dir=requirements_directory, - index=index_url, - extra_indexes=extra_index_url, - pypi_mirror=pypi_mirror, - ) - if not c.ok: - sp.write_err(vistir.compat.fs_str( - "{0}: {1}".format( - crayons.red("WARNING"), - "Failed installing package {0}".format(pkg_line) - ), - )) - sp.write_err(vistir.compat.fs_str( - "Error text: {0}".format(c.out) - )) - raise RuntimeError(c.err) + if deps: + to_install.extend([ + Requirement.from_line(d) for d in list(deps[0].values()) + ]) + no_deps = True + for dep in to_install: + sp.text = "Installing {0}...".format(dep.name) + c = pip_install( + dep, + ignore_hashes=True, + allow_global=system, + selective_upgrade=selective_upgrade, + no_deps=no_deps, + pre=pre, + requirements_dir=requirements_directory, + index=index_url, + extra_indexes=extra_index_url, + pypi_mirror=pypi_mirror, + ) + if not c.ok: + sp.write_err(vistir.compat.fs_str( + "{0}: {1}".format( + crayons.red("WARNING"), + "Failed installing package {0}".format(pkg_line) + ), + )) + sp.write_err(vistir.compat.fs_str( + "Error text: {0}".format(c.out) + )) + raise RuntimeError(c.err) + if environments.is_verbose(): + click.echo(crayons.blue(format_pip_output(c.out))) except (ValueError, RuntimeError) as e: sp.write_err(vistir.compat.fs_str( "{0}: {1}".format(crayons.red("WARNING"), e), @@ -2105,7 +2140,7 @@ def do_install( )) sys.exit(1) # Warn if --editable wasn't passed. - if pkg_requirement.is_vcs and not pkg_requirement.editable: + if pkg_requirement.is_vcs and not pkg_requirement.editable and not PIPENV_RESOLVE_VCS: sp.write_err( "{0}: You installed a VCS dependency in non-editable mode. " "This will work fine, but sub-dependencies will not be resolved by {1}." @@ -2115,24 +2150,23 @@ def do_install( crayons.red("$ pipenv lock"), ) ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. - if c.return_code != 0: + # Ensure that package was successfully installed. + if c.return_code != 0: + sp.write_err(vistir.compat.fs_str( + "{0} An error occurred while installing {1}!".format( + crayons.red("Error: ", bold=True), crayons.green(pkg_line) + ), + )) + sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) + if "setup.py egg_info" in c.err: sp.write_err(vistir.compat.fs_str( - "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(pkg_line) - ), + "This is likely caused by a bug in {0}. " + "Report this to its maintainers.".format( + crayons.green(pkg_requirement.name) + ) )) - sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) - if "setup.py egg_info" in c.err: - sp.write_err(vistir.compat.fs_str( - "This is likely caused by a bug in {0}. " - "Report this to its maintainers.".format( - crayons.green(pkg_requirement.name) - ) - )) - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) - sys.exit(1) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sys.exit(1) sp.write(vistir.compat.fs_str( u"{0} {1} {2} {3}{4}".format( crayons.normal(u"Adding", bold=True), diff --git a/pipenv/environment.py b/pipenv/environment.py index 49cad91d..a8489d95 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -248,7 +248,6 @@ class Environment(object): "prefix='{1}');paths['platlib'] = distutils.sysconfig.get_python_lib(" "plat_specific=1, prefix='{1}');print(json.dumps(paths))" ) - vistir.misc.echo("command: {0}".format(py_command.format(install_scheme, prefix)), fg="white", style="bold", err=True) command = [self.python, "-c", py_command.format(install_scheme, prefix)] c = vistir.misc.run( command, return_object=True, block=True, nospin=True, write_to_stdout=False @@ -368,7 +367,9 @@ class Environment(object): @contextlib.contextmanager def get_finder(self, pre=False): - from .vendor.pip_shims.shims import Command, cmdoptions, index_group, PackageFinder + from .vendor.pip_shims.shims import ( + Command, cmdoptions, index_group, PackageFinder, parse_version, pip_version + ) from .environments import PIPENV_CACHE_DIR index_urls = [source.get("url") for source in self.sources] @@ -387,25 +388,35 @@ class Environment(object): pip_options.cache_dir = PIPENV_CACHE_DIR pip_options.pre = self.pipfile.get("pre", pre) with pip_command._build_session(pip_options) as session: - finder = PackageFinder( - find_links=pip_options.find_links, - index_urls=index_urls, allow_all_prereleases=pip_options.pre, - trusted_hosts=pip_options.trusted_hosts, - process_dependency_links=pip_options.process_dependency_links, - session=session - ) + finder_args = { + "find_links": pip_options.find_links, + "index_urls": index_urls, + "allow_all_prereleases": pip_options.pre, + "trusted_hosts": pip_options.trusted_hosts, + "session": session + } + if parse_version(pip_version) < parse_version("19.0"): + finder_args.update( + {"process_dependency_links": pip_options.process_dependency_links} + ) + finder = PackageFinder(**finder_args) yield finder def get_package_info(self, pre=False): + from .vendor.pip_shims.shims import pip_version, parse_version dependency_links = [] packages = self.get_installed_packages() # This code is borrowed from pip's current implementation - for dist in packages: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) + if parse_version(pip_version) < parse_version("19.0"): + for dist in packages: + if dist.has_metadata('dependency_links.txt'): + dependency_links.extend( + dist.get_metadata_lines('dependency_links.txt') + ) with self.get_finder() as finder: - finder.add_dependency_links(dependency_links) + if parse_version(pip_version) < parse_version("19.0"): + finder.add_dependency_links(dependency_links) for dist in packages: typ = 'unknown' @@ -433,7 +444,7 @@ class Environment(object): def get_outdated_packages(self, pre=False): return [ pkg for pkg in self.get_package_info(pre=pre) - if pkg.latest_version._version > pkg.parsed_version._version + if pkg.latest_version._key > pkg.parsed_version._key ] @classmethod diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 20a0796f..1219cc24 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -97,6 +97,7 @@ class Entry(object): def __init__(self, name, entry_dict, project, resolver, reverse_deps=None, dev=False): super(Entry, self).__init__() + from pipenv.vendor.requirementslib.models.utils import tomlkit_value_to_python self.name = name if isinstance(entry_dict, dict): self.entry_dict = self.clean_initial_dict(entry_dict) @@ -106,7 +107,9 @@ class Entry(object): section = "develop" if dev else "default" pipfile_section = "dev-packages" if dev else "packages" self.dev = dev - self.pipfile = project.parsed_pipfile.get(pipfile_section, {}) + self.pipfile = tomlkit_value_to_python( + project.parsed_pipfile.get(pipfile_section, {}) + ) self.lockfile = project.lockfile_content.get(section, {}) self.pipfile_dict = self.pipfile.get(self.pipfile_name, {}) if self.dev and self.name in project.lockfile_content.get("default", {}): diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py index 32716bea..9320a437 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import sys -__version__ = '0.3.3' +__version__ = "0.3.3" from . import shims diff --git a/pipenv/vendor/pip_shims/shims.py b/pipenv/vendor/pip_shims/shims.py index 5ea2691f..4cad473a 100644 --- a/pipenv/vendor/pip_shims/shims.py +++ b/pipenv/vendor/pip_shims/shims.py @@ -15,7 +15,7 @@ from six.moves import Callable # type: ignore # noqa # isort:skip class _shims(object): - CURRENT_PIP_VERSION = "19.0.3" + CURRENT_PIP_VERSION = "19.1.1" BASE_IMPORT_PATH = os.environ.get("PIP_SHIMS_BASE_MODULE", "pip") path_info = namedtuple("PathInfo", "path start_version end_version") @@ -141,14 +141,25 @@ class _shims(object): ), "Link": ("index.Link", "7.0.0", "9999"), "make_abstract_dist": ( - ("operations.prepare.make_abstract_dist", "10.0.0", "9999"), + ( + "distributions.make_distribution_for_install_requirement", + "19.1.2", + "9999", + ), + ("operations.prepare.make_abstract_dist", "10.0.0", "19.1.1"), ("req.req_set.make_abstract_dist", "7.0.0", "9.0.3"), ), + "make_distribution_for_install_requirement": ( + "distributions.make_distribution_for_install_requirement", + "19.1.2", + "9999", + ), "make_option_group": ( ("cli.cmdoptions.make_option_group", "18.1", "9999"), ("cmdoptions.make_option_group", "7.0.0", "18.0"), ), "PackageFinder": ("index.PackageFinder", "7.0.0", "9999"), + "CandidateEvaluator": ("index.CandidateEvaluator", "19.1", "9999"), "parse_requirements": ("req.req_file.parse_requirements", "7.0.0", "9999"), "path_to_url": ("download.path_to_url", "7.0.0", "9999"), "PipError": ("exceptions.PipError", "7.0.0", "9999"), @@ -159,18 +170,44 @@ class _shims(object): ), "RequirementSet": ("req.req_set.RequirementSet", "7.0.0", "9999"), "RequirementTracker": ("req.req_tracker.RequirementTracker", "7.0.0", "9999"), - "Resolver": ("resolve.Resolver", "7.0.0", "9999"), + "Resolver": ( + ("resolve.Resolver", "7.0.0", "19.1.1"), + ("legacy_resolve.Resolver", "19.1.2", "9999"), + ), "SafeFileCache": ("download.SafeFileCache", "7.0.0", "9999"), "UninstallPathSet": ("req.req_uninstall.UninstallPathSet", "7.0.0", "9999"), "url_to_path": ("download.url_to_path", "7.0.0", "9999"), "USER_CACHE_DIR": ("locations.USER_CACHE_DIR", "7.0.0", "9999"), - "VcsSupport": ("vcs.VcsSupport", "7.0.0", "9999"), + "VcsSupport": ( + ("vcs.VcsSupport", "7.0.0", "19.1.1"), + ("vcs.versioncontrol.VcsSupport", "19.2", "9999"), + ), "Wheel": ("wheel.Wheel", "7.0.0", "9999"), "WheelCache": ( ("cache.WheelCache", "10.0.0", "9999"), ("wheel.WheelCache", "7", "9.0.3"), ), "WheelBuilder": ("wheel.WheelBuilder", "7.0.0", "9999"), + "AbstractDistribution": ( + "distributions.base.AbstractDistribution", + "19.1.2", + "9999", + ), + "InstalledDistribution": ( + "distributions.installed.InstalledDistribution", + "19.1.2", + "9999", + ), + "SourceDistribution": ( + ("req.req_set.IsSDist", "7.0.0", "9.0.3"), + ("operations.prepare.IsSDist", "10.0.0", "19.1.1"), + ("distributions.source.SourceDistribution", "19.1.2", "9999"), + ), + "WheelDistribution": ( + "distributions.wheel.WheelDistribution", + "19.1.2", + "9999", + ), "PyPI": ("models.index.PyPI", "7.0.0", "9999"), "stdlib_pkgs": ( ("utils.compat.stdlib_pkgs", "18.1", "9999"), diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 74429dbc..0f202614 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -194,11 +194,15 @@ WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() class _Pipfile(object): def __init__(self, path): self.path = path - self.document = tomlkit.document() - self.document["source"] = tomlkit.aot() - self.document["requires"] = tomlkit.table() - self.document["packages"] = tomlkit.table() - self.document["dev_packages"] = tomlkit.table() + if self.path.exists(): + self.loads() + else: + self.document = tomlkit.document() + self.document["source"] = self.document.get("source", tomlkit.aot()) + self.document["requires"] = self.document.get("requires", tomlkit.table()) + self.document["packages"] = self.document.get("packages", tomlkit.table()) + self.document["dev_packages"] = self.document.get("dev_packages", tomlkit.table()) + super(_Pipfile, self).__init__() def install(self, package, value, dev=False): section = "packages" if not dev else "dev_packages" @@ -210,15 +214,27 @@ class _Pipfile(object): self.document[section][package] = value self.write() + def remove(self, package, dev=False): + section = "packages" if not dev else "dev_packages" + if not dev and package not in self.document[section]: + if package in self.document["dev_packages"]: + section = "dev_packages" + del self.document[section][package] + self.write() + def add(self, package, value, dev=False): self.install(package, value, dev=dev) + def update(self, package, value, dev=False): + self.install(package, value, dev=dev) + def loads(self): self.document = tomlkit.loads(self.path.read_text()) def dumps(self): source_table = tomlkit.table() - source_table["url"] = os.environ.get("PIPENV_TEST_INDEX") + pypi_url = os.environ.get("PIPENV_PYPI_URL", "https://pypi.org/simple") + source_table["url"] = os.environ.get("PIPENV_TEST_INDEX", pypi_url) source_table["verify_ssl"] = False source_table["name"] = "pipenv_test_index" self.document["source"].append(source_table) diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index ddf54117..a231c17d 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -372,3 +372,18 @@ def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, testsr c = p.pipenv('run python -c "import requests, flask, six, jinja2"') assert c.return_code == 0, c.err + + +@pytest.mark.outdated +@pytest.mark.py3_only +def test_outdated_should_compare_postreleases_without_failing(PipenvInstance): + with PipenvInstance(chdir=True) as p: + c = p.pipenv("install ibm-db-sa-py3==0.3.0") + assert c.return_code == 0 + c = p.pipenv("update --outdated") + assert c.return_code == 0 + assert "Skipped Update" in c.err + p._pipfile.update("ibm-db-sa-py3", "*") + c = p.pipenv("update --outdated") + assert c.return_code != 0 + assert "out-of-date" in c.out