From 8c8d3d1f8e0acee6b22a09f9ae9d80b093d4e7a1 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Wed, 1 Mar 2023 04:26:36 -0500 Subject: [PATCH] Improve pipenv update and add pipenv upgrade command (#5617) * Split apart core using pycharm refactor move methods. * move init to remove cicular import. * Fix imports. * Check in concept for pipenv upgrade command * Fix upgrade command expectation on how it updates the lockfile. * Actually write the result to the Pipfile, and fix secondary bug with items not being written to the Pipfile. * Fix issue where package being upgraded already exists. * Add news fragment. * Integrate upgrade with a refactor of update. * Handle cases where there is nothing to upgrade. * Add lock-only option. --- Pipfile.lock | 1 + news/5617.feature.rst | 4 + pipenv/cli/command.py | 105 ++++++------- pipenv/cli/options.py | 33 +++- pipenv/project.py | 11 +- pipenv/routines/lock.py | 2 - pipenv/routines/update.py | 191 +++++++++++++++++++++++ pipenv/utils/project.py | 2 +- pipenv/utils/resolver.py | 8 +- tests/integration/test_install_twists.py | 23 +-- 10 files changed, 297 insertions(+), 83 deletions(-) create mode 100644 news/5617.feature.rst create mode 100644 pipenv/routines/update.py diff --git a/Pipfile.lock b/Pipfile.lock index 753d6539..7ff8e123 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -693,6 +693,7 @@ "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa", "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf" ], + "index": "pypi", "markers": "python_version >= '3.7' and python_version < '4'", "version": "==2.28.2" }, diff --git a/news/5617.feature.rst b/news/5617.feature.rst new file mode 100644 index 00000000..fc47427f --- /dev/null +++ b/news/5617.feature.rst @@ -0,0 +1,4 @@ +Provide a more powerful solution than ``--keep-outdated`` and ``--selective-upgrade`` which are deprecated for removal. +Introducing the ``pipenv upgrade`` command which takes the same package specifiers as ``pipenv install`` and +updates the ``Pipfile`` and ``Pipfile.lock`` with a valid lock resolution that only effects the specified packages and their dependencies. +Additionally, the ``pipenv update`` command has been updated to use the ``pipenv upgrade`` routine when packages are provided, which will install sync the new lock file as well. diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index ba369e29..a0417332 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -20,6 +20,7 @@ from pipenv.cli.options import ( sync_options, system_option, uninstall_options, + upgrade_options, verbose_option, ) from pipenv.utils.dependencies import get_lockfile_section_using_pipfile_category @@ -253,6 +254,39 @@ def install(state, **kwargs): ) +@cli.command( + short_help="Resolves provided packages and adds them to Pipfile, or (if no packages are given), merges results to Pipfile.lock", + context_settings=subcommand_context, +) +@system_option +@site_packages_option +@install_options +@upgrade_options +@pass_state +def upgrade(state, **kwargs): + from pipenv.routines.update import upgrade + from pipenv.utils.project import ensure_project + + ensure_project( + state.project, + python=state.python, + pypi_mirror=state.pypi_mirror, + warn=(not state.quiet), + site_packages=state.site_packages, + clear=state.clear, + ) + + upgrade( + state.project, + pre=state.installstate.pre, + packages=state.installstate.packages, + editable_packages=state.installstate.editables, + categories=state.installstate.categories, + system=state.system, + lock_only=state.installstate.lock_only, + ) + + @cli.command( short_help="Uninstalls a provided package and removes it from Pipfile.", context_settings=subcommand_context, @@ -332,7 +366,6 @@ def lock(ctx, state, **kwargs): pre = state.installstate.pre do_lock( state.project, - ctx=ctx, clear=state.clear, pre=pre, keep_outdated=state.installstate.keep_outdated, @@ -534,76 +567,32 @@ def check( @option("--outdated", is_flag=True, default=False, help="List out-of-date dependencies.") @option("--dry-run", is_flag=True, default=None, help="List out-of-date dependencies.") @install_options +@upgrade_options @pass_state @pass_context def update(ctx, state, bare=False, dry_run=None, outdated=False, **kwargs): - """Runs lock, then sync.""" - from pipenv.routines.install import do_sync - from pipenv.routines.lock import do_lock - from pipenv.routines.outdated import do_outdated - from pipenv.utils.project import ensure_project + """Runs lock when no packages are specified, or upgrade, and then sync.""" + from pipenv.routines.update import do_update - ensure_project( + do_update( state.project, python=state.python, - pypi_mirror=state.pypi_mirror, - warn=(not state.quiet), site_packages=state.site_packages, clear=state.clear, - ) - if not outdated: - outdated = bool(dry_run) - if outdated: - do_outdated( - state.project, - clear=state.clear, - pre=state.installstate.pre, - pypi_mirror=state.pypi_mirror, - ) - packages = [p for p in state.installstate.packages if p] - editable = [p for p in state.installstate.editables if p] - if not packages: - echo( - "{} {} {} {}{}".format( - style("Running", bold=True), - style("$ pipenv lock", fg="yellow", bold=True), - style("then", bold=True), - style("$ pipenv sync", fg="yellow", bold=True), - style(".", bold=True), - ) - ) - else: - for package in packages + editable: - if package not in state.project.all_packages: - echo( - "{}: {} was not found in your Pipfile! Aborting." - "".format( - style("Warning", fg="red", bold=True), - style(package, fg="green", bold=True), - ), - err=True, - ) - ctx.abort() - do_lock( - state.project, - ctx=ctx, - clear=state.clear, pre=state.installstate.pre, + pypi_mirror=state.pypi_mirror, keep_outdated=state.installstate.keep_outdated, - pypi_mirror=state.pypi_mirror, - write=not state.quiet, - ) - do_sync( - state.project, + system=False, + packages=state.installstate.packages, + editable_packages=state.installstate.editables, dev=state.installstate.dev, - python=state.python, bare=bare, - dont_upgrade=not state.installstate.keep_outdated, - user=False, - clear=state.clear, - unused=False, - pypi_mirror=state.pypi_mirror, extra_pip_args=state.installstate.extra_pip_args, + categories=state.installstate.categories, + quiet=state.quiet, + dry_run=dry_run, + outdated=outdated, + lock_only=state.installstate.lock_only, ) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 8ac633c0..d6983119 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -159,7 +159,7 @@ def keep_outdated_option(f): click.secho( "The flag --keep-outdated has been deprecated for removal. " "The flag does not respect package resolver results and leads to inconsistent lock files. " - "Please pin relevant requirements in your Pipfile and discontinue use of this flag.", + "Consider using the new `pipenv upgrade` command to selectively upgrade packages.", fg="yellow", bold=True, err=True, @@ -182,6 +182,15 @@ def selective_upgrade_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) state.installstate.selective_upgrade = value + if value: + click.secho( + "The flag --selective-upgrade has been deprecated for removal. " + "The flag is buggy and leads to inconsistent lock files. " + "Consider using the new `pipenv upgrade` command to selectively upgrade packages.", + fg="yellow", + bold=True, + err=True, + ) return value return option( @@ -503,6 +512,23 @@ def deploy_option(f): )(f) +def lock_only_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.lock_only = value + return value + + return option( + "--lock-only", + is_flag=True, + default=False, + help="Only update lock file (specifiers not added to Pipfile).", + callback=callback, + type=click_types.BOOL, + expose_value=False, + )(f) + + def setup_verbosity(ctx, param, value): if not value: return @@ -586,6 +612,11 @@ def install_options(f): return f +def upgrade_options(f): + f = lock_only_option(f) + return f + + def general_options(f): f = common_options(f) f = site_packages_option(f) diff --git a/pipenv/project.py b/pipenv/project.py index 1710abcf..802c144e 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -26,7 +26,6 @@ from pipenv.utils.constants import is_type_checking from pipenv.utils.dependencies import ( get_canonical_names, is_editable, - is_star, pep423_name, python_version, ) @@ -977,12 +976,12 @@ class Project: # Set empty group if it doesn't exist yet. if category not in p: p[category] = {} - name = self.get_package_name_in_pipfile(req_name, category=category) - if name and is_star(converted): - # Skip for wildcard version - return # Add the package to the group. - p[category][name or pep423_name(req_name)] = converted + name = self.get_package_name_in_pipfile(req_name, category=category) + normalized_name = pep423_name(req_name) + if name and name != normalized_name: + self.remove_package_from_pipfile(name, category=category) + p[category][normalized_name] = converted # Write Pipfile. self.write_toml(p) diff --git a/pipenv/routines/lock.py b/pipenv/routines/lock.py index b2cc4fc9..f81833c9 100644 --- a/pipenv/routines/lock.py +++ b/pipenv/routines/lock.py @@ -10,7 +10,6 @@ from pipenv.vendor import click def do_lock( project, - ctx=None, system=False, clear=False, pre=False, @@ -27,7 +26,6 @@ def do_lock( if not project.lockfile_exists: raise exceptions.PipenvOptionsError( "--keep-outdated", - ctx=ctx, message="Pipfile.lock must exist to use --keep-outdated!", ) cached_lockfile = project.lockfile_content diff --git a/pipenv/routines/update.py b/pipenv/routines/update.py new file mode 100644 index 00000000..ca9f6aec --- /dev/null +++ b/pipenv/routines/update.py @@ -0,0 +1,191 @@ +import sys + +from pipenv.routines.install import do_sync +from pipenv.routines.lock import do_lock +from pipenv.routines.outdated import do_outdated +from pipenv.utils.dependencies import ( + convert_deps_to_pip, + get_pipfile_category_using_lockfile_section, + is_star, +) +from pipenv.utils.project import ensure_project +from pipenv.utils.resolver import venv_resolve_deps +from pipenv.vendor import click +from pipenv.vendor.requirementslib.models.requirements import Requirement + + +def do_update( + project, + python=None, + pre=False, + system=False, + packages=None, + editable_packages=None, + site_packages=False, + pypi_mirror=None, + dev=False, + categories=None, + extra_pip_args=None, + quiet=False, + bare=False, + dry_run=None, + outdated=False, + keep_outdated=False, + clear=False, + lock_only=False, +): + ensure_project( + project, + python=python, + pypi_mirror=pypi_mirror, + warn=(not quiet), + site_packages=site_packages, + clear=clear, + ) + if not outdated: + outdated = bool(dry_run) + if outdated: + do_outdated( + project, + clear=clear, + pre=pre, + pypi_mirror=pypi_mirror, + ) + packages = [p for p in packages if p] + editable = [p for p in editable_packages if p] + if not packages: + click.echo( + "{} {} {} {}{}".format( + click.style("Running", bold=True), + click.style("$ pipenv lock", fg="yellow", bold=True), + click.style("then", bold=True), + click.style("$ pipenv sync", fg="yellow", bold=True), + click.style(".", bold=True), + ) + ) + do_lock( + project, + clear=clear, + pre=pre, + keep_outdated=keep_outdated, + pypi_mirror=pypi_mirror, + write=not quiet, + ) + else: + upgrade( + project, + pre=pre, + system=system, + packages=packages, + editable_packages=editable, + pypi_mirror=pypi_mirror, + categories=categories, + lock_only=lock_only, + ) + + do_sync( + project, + dev=dev, + categories=categories, + python=python, + bare=bare, + dont_upgrade=not keep_outdated, + user=False, + clear=clear, + unused=False, + pypi_mirror=pypi_mirror, + extra_pip_args=extra_pip_args, + ) + + +def upgrade( + project, + pre=False, + system=False, + packages=None, + editable_packages=None, + pypi_mirror=None, + categories=None, + lock_only=False, +): + + lockfile = project._lockfile() + if not pre: + pre = project.settings.get("allow_prereleases") + if not categories: + categories = ["default"] + + package_args = [p for p in packages] + [f"-e {pkg}" for pkg in editable_packages] + + reqs = {} + requested_packages = {} + for package in package_args[:]: + # section = project.packages if not dev else project.dev_packages + section = {} + package = Requirement.from_line(package) + package_name, package_val = package.pipfile_entry + requested_packages[package_name] = package + try: + if not is_star(section[package_name]) and is_star(package_val): + # Support for VCS dependencies. + package_val = convert_deps_to_pip( + {package_name: section[package_name]}, project=project + )[0] + except KeyError: + pass + reqs[package_name] = package_val + + if not reqs: + click.echo("Nothing to upgrade!") + sys.exit(0) + + # Resolve package to generate constraints of new package data + upgrade_lock_data = venv_resolve_deps( + reqs, + which=project._which, + project=project, + lockfile={}, + category="default", + pre=pre, + allow_global=system, + pypi_mirror=pypi_mirror, + keep_outdated=False, + ) + if not upgrade_lock_data: + click.echo("Nothing to upgrade!") + sys.exit(0) + + # Upgrade the relevant packages in the various categories specified + for category in categories: + pipfile_category = get_pipfile_category_using_lockfile_section(category) + if project.pipfile_exists: + packages = project.parsed_pipfile.get(pipfile_category, {}) + else: + packages = project.get_pipfile_section(pipfile_category) + for package_name, requirement in requested_packages.items(): + requested_package = reqs[package_name] + if package_name not in packages: + packages.append(package_name, requested_package) + else: + packages[package_name] = requested_package + if lock_only is False: + project.add_package_to_pipfile(requirement, category=pipfile_category) + + full_lock_resolution = venv_resolve_deps( + packages, + which=project._which, + project=project, + lockfile={}, + category=pipfile_category, + pre=pre, + allow_global=system, + pypi_mirror=pypi_mirror, + keep_outdated=False, + ) + # Mutate the existing lockfile with the upgrade data for the categories + for package_name, _ in upgrade_lock_data.items(): + correct_package_lock = full_lock_resolution[package_name] + lockfile[category][package_name] = correct_package_lock + + lockfile.update({"_meta": project.get_lockfile_meta()}) + project.write_lockfile(lockfile) diff --git a/pipenv/utils/project.py b/pipenv/utils/project.py index 29c936b5..f7791ed9 100644 --- a/pipenv/utils/project.py +++ b/pipenv/utils/project.py @@ -75,5 +75,5 @@ def ensure_project( project, validate=validate, skip_requirements=skip_requirements, - system=system_or_exists, + system=system, ) diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index f0283218..7fdbaec4 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -973,8 +973,8 @@ def venv_resolve_deps( :param Dict[str, Any] lockfile: A project lockfile to mutate, defaults to None :param bool keep_outdated: Whether to retain outdated dependencies and resolve with them in mind, defaults to False :raises RuntimeError: Raised on resolution failure - :return: Nothing - :rtype: None + :return: The lock data + :rtype: dict """ from pipenv import resolver @@ -989,7 +989,7 @@ def venv_resolve_deps( if not pipfile: pipfile = getattr(project, category, {}) - if not lockfile: + if lockfile is None: lockfile = project._lockfile(categories=[category]) req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") cmd = [ @@ -1063,7 +1063,7 @@ def venv_resolve_deps( os.unlink(target_file.name) if lockfile_section not in lockfile: lockfile[lockfile_section] = {} - prepare_lockfile(results, pipfile, lockfile[lockfile_section]) + return prepare_lockfile(results, pipfile, lockfile[lockfile_section]) def resolve_deps( diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 5b62a0aa..e13ef8d0 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -117,14 +117,15 @@ Requests = "==2.14.0" # Inline comment assert p.pipfile["packages"]["Requests"] == "==2.14.0" c = p.pipenv("install requests==2.18.4") assert c.returncode == 0 - assert p.pipfile["packages"]["Requests"] == "==2.18.4" + assert "Requests" not in p.pipfile["packages"] + assert "requests" in p.pipfile["packages"] + assert p.pipfile["packages"]["requests"] == "==2.18.4" c = p.pipenv("install python_DateUtil") assert c.returncode == 0 assert "python-dateutil" in p.pipfile["packages"] with open(p.pipfile_path) as f: contents = f.read() assert "# Pre comment" in contents - assert "# Inline comment" in contents @pytest.mark.eggs @@ -147,24 +148,24 @@ def test_local_package(pipenv_instance_private_pypi, pip_src_dir, testsroot): with tarfile.open(copy_to, "r:gz") as tgz: def is_within_directory(directory, target): - + abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) - + prefix = os.path.commonprefix([abs_directory, abs_target]) - + return prefix == abs_directory - + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): - + for member in tar.getmembers(): member_path = os.path.join(path, member.name) if not is_within_directory(path, member_path): raise Exception("Attempted Path Traversal in Tar File") - - tar.extractall(path, members, numeric_owner) - - + + tar.extractall(path, members, numeric_owner) + + safe_extract(tgz, path=p.path) c = p.pipenv(f"install -e {package}") assert c.returncode == 0