From 592cbe54a12064df0d069c55336cc2ee6ee1bda0 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 10 Apr 2020 15:51:54 +1000 Subject: [PATCH 1/7] Issue #3316: Include all deps in 'pipenv lock -r --dev' * Implements PEEP-006 * `pipenv lock -r --dev` is now consistent with other commands and the CLI help output, and includes both default and dev dependencies in the result * New `--dev-only` option allows requesting the previous behaviour (which was specifically designed to support the traditional `requirements.txt`/`dev-requirements.txt` split) --- docs/advanced.rst | 50 ++++++++++++++----- news/3316.feature.rst | 5 ++ pipenv/cli/command.py | 17 +++++-- pipenv/cli/options.py | 18 ++++++- pipenv/core.py | 40 ++++++++------- .../vendor/requirementslib/models/lockfile.py | 2 + 6 files changed, 95 insertions(+), 37 deletions(-) create mode 100644 news/3316.feature.rst diff --git a/docs/advanced.rst b/docs/advanced.rst index e6e31f62..9fc2470b 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -149,7 +149,9 @@ Anaconda uses Conda to manage packages. To reuse Conda–installed Python packag ☤ Generating a ``requirements.txt`` ----------------------------------- -You can convert a ``Pipfile`` and ``Pipfile.lock`` into a ``requirements.txt`` file very easily, and get all the benefits of extras and other goodies we have included. +You can convert a ``Pipfile`` and ``Pipfile.lock`` into a ``requirements.txt`` +file very easily, and get all the benefits of extras and other goodies we have +included. Let's take this ``Pipfile``:: @@ -160,7 +162,10 @@ Let's take this ``Pipfile``:: [packages] requests = {version="*"} -And generate a ``requirements.txt`` out of it:: + [dev-packages] + pytest = {version="*"} + +And generate a set of requirements out of it with only the production dependencies:: $ pipenv lock -r chardet==3.0.4 @@ -169,22 +174,41 @@ And generate a ``requirements.txt`` out of it:: idna==2.6 urllib3==1.22 -If you wish to generate a ``requirements.txt`` with only the development requirements you can do that too! Let's take the following ``Pipfile``:: - - [[source]] - url = "https://pypi.python.org/simple" - verify_ssl = true - - [dev-packages] - pytest = {version="*"} - -And generate a ``requirements.txt`` out of it:: +As with other commands, passing ``--dev`` will include both the production and +development dependencies:: $ pipenv lock -r --dev + chardet==3.0.4 + requests==2.18.4 + certifi==2017.7.27.1 + idna==2.6 + urllib3==1.22 + py==1.4.34 + pytest==3.2.3 + +Finally, if you wish to generate a requirements file with only the +development requirements you can do that too, using the ``--dev-only`` +flag:: + + $ pipenv lock -r --dev-only + py==1.4.34 + pytest==3.2.3 + +The locked requirements are written to stdout, with shell output redirection +used to write them to a file:: + + $ pipenv lock -r > requirements.txt + $ pipenv lock -r --dev-only > dev-requirements.txt + $ cat requirements.txt + chardet==3.0.4 + requests==2.18.4 + certifi==2017.7.27.1 + idna==2.6 + urllib3==1.22 + $ cat dev-requirements.txt py==1.4.34 pytest==3.2.3 -Very fancy. ☤ Detection of Security Vulnerabilities --------------------------------------- diff --git a/news/3316.feature.rst b/news/3316.feature.rst new file mode 100644 index 00000000..932fd759 --- /dev/null +++ b/news/3316.feature.rst @@ -0,0 +1,5 @@ +For consistency with other commands and the ``-dev`` option +description, ``pipenv lock --requirements --dev`` now emits +both default and development dependencies. +A new ``--dev-only`` has been added to request to previous +behaviour (e.g. to generate a ``dev-requirements.txt`` file). diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 9d3ce9bf..c6f1e7ec 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -237,7 +237,7 @@ def install( lock=not state.installstate.skip_lock, ignore_pipfile=state.installstate.ignore_pipfile, skip_lock=state.installstate.skip_lock, - requirements=state.installstate.requirementstxt, + requirementstxt=state.installstate.requirementstxt, sequential=state.installstate.sequential, pre=state.installstate.pre, code=state.installstate.code, @@ -317,13 +317,24 @@ def lock( three=state.three, python=state.python, pypi_mirror=state.pypi_mirror, warn=(not state.quiet), site_packages=state.site_packages, ) - if state.installstate.requirementstxt: + if state.lockoptions.emit_requirements: + # Setting "requirements=True" means do_init() just emits the + # install requirements file to stdout, it doesn't install anything + if state.installstate.dev: + pass # TODO: Emit behaviour change warning as per PEEP 006 do_init( dev=state.installstate.dev, - requirements=state.installstate.requirementstxt, + dev_only=state.lockoptions.dev_only, + emit_requirements=state.lockoptions.emit_requirements, pypi_mirror=state.pypi_mirror, pre=state.installstate.pre, ) + elif state.lockoptions.dev_only: + raise exceptions.PipenvOptionsError( + "--dev-only", + "--dev-only is only permitted in combination with --requirements. " + "Aborting." + ) do_lock( ctx=ctx, clear=state.clear, diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index fc45256f..9c5c1a18 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -65,6 +65,7 @@ class State(object): self.clear = False self.system = False self.installstate = InstallState() + self.lockoptions = LockOptions() class InstallState(object): @@ -82,6 +83,10 @@ class InstallState(object): self.packages = [] self.editables = [] +class LockOptions(object): + def __init__(self): + self.dev_only = False + self.emit_requirements = False pass_state = make_pass_decorator(State, ensure=True) @@ -300,15 +305,23 @@ def requirementstxt_option(f): help="Import a requirements.txt file.", callback=callback)(f) -def requirements_flag(f): +def emit_requirements_flag(f): def callback(ctx, param, value): state = ctx.ensure_object(State) if value: - state.installstate.requirementstxt = value + state.lockoptions.emit_requirements = value return value return option("--requirements", "-r", default=False, is_flag=True, expose_value=False, help="Generate output in requirements.txt format.", callback=callback)(f) +def dev_only_flag(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + state.lockoptions.dev_only = value + return value + return option("--dev-only", default=False, is_flag=True, expose_value=False, + help="Emit development dependencies *only* (overrides --dev)", callback=callback)(f) def code_option(f): def callback(ctx, param, value): @@ -397,6 +410,7 @@ def uninstall_options(f): def lock_options(f): f = install_base_options(f) f = requirements_flag(f) + f = dev_only_flag(f) return f diff --git a/pipenv/core.py b/pipenv/core.py index 74b72359..54ad73a5 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -772,9 +772,9 @@ def batch_install(deps_list, procs, failed_deps_queue, def do_install_dependencies( dev=False, - only=False, + dev_only=False, bare=False, - requirements=False, + emit_requirements=False, allow_global=False, ignore_hashes=False, skip_lock=False, @@ -785,11 +785,11 @@ def do_install_dependencies( """" Executes the install functionality. - If requirements is True, simply spits out a requirements format to stdout. + If emit_requirements is True, simply spits out a requirements format to stdout. """ from six.moves import queue - if requirements: + if emit_requirements: bare = True # Load the lockfile if it exists, or if only is being used (e.g. lock is being used). if skip_lock or only or not project.lockfile_exists: @@ -812,14 +812,14 @@ def do_install_dependencies( ) # Allow pip to resolve dependencies when in skip-lock mode. no_deps = not skip_lock # skip_lock true, no_deps False, pip resolves deps - deps_list = list(lockfile.get_requirements(dev=dev, only=requirements)) - if requirements: + dev = dev or dev_only + deps_list = list(lockfile.get_requirements(dev=dev, only=dev_only)) + if emit_requirements: index_args = prepare_pip_source_args(project.sources) index_args = " ".join(index_args).replace(" -", "\n-") deps = [ req.as_line(sources=False, include_hashes=False) for req in deps_list ] - # Output only default dependencies click.echo(index_args) click.echo( "\n".join(sorted(deps)) @@ -1171,7 +1171,8 @@ def do_purge(bare=False, downloads=False, allow_global=False): def do_init( dev=False, - requirements=False, + dev_only=False, + emit_requirements=False, allow_global=False, ignore_pipfile=False, skip_lock=False, @@ -1276,7 +1277,8 @@ def do_init( ) do_install_dependencies( dev=dev, - requirements=requirements, + dev_only=dev_only, + emit_requirements=emit_requirements, allow_global=allow_global, skip_lock=skip_lock, concurrent=concurrent, @@ -1855,7 +1857,7 @@ def do_install( lock=True, ignore_pipfile=False, skip_lock=False, - requirements=False, + requirementstxt=False, sequential=False, pre=False, code=False, @@ -1878,7 +1880,7 @@ def do_install( package_args = [p for p in packages if p] + [p for p in editable_packages if p] skip_requirements = False # Don't search for requirements.txt files if the user provides one - if requirements or package_args or project.pipfile_exists: + if requirementstxt or package_args or project.pipfile_exists: skip_requirements = True concurrent = not sequential # Ensure that virtualenv is available and pipfile are available @@ -1903,7 +1905,7 @@ def do_install( pre = project.settings.get("allow_prereleases") if not keep_outdated: keep_outdated = project.settings.get("keep_outdated") - remote = requirements and is_valid_url(requirements) + remote = requirementstxt and is_valid_url(requirementstxt) # Warn and exit if --system is used without a pipfile. if (system and package_args) and not (PIPENV_VIRTUALENV): raise exceptions.SystemUsageError @@ -1922,17 +1924,17 @@ def do_install( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory ) temp_reqs = fd.name - requirements_url = requirements + requirements_url = requirementstxt # Download requirements file try: - download_file(requirements, temp_reqs) + download_file(requirements_url, temp_reqs) except IOError: fd.close() os.unlink(temp_reqs) click.echo( crayons.red( u"Unable to find requirements file at {0}.".format( - crayons.normal(requirements) + crayons.normal(requirements_url) ) ), err=True, @@ -1941,9 +1943,9 @@ def do_install( finally: fd.close() # Replace the url with the temporary requirements file - requirements = temp_reqs + requirementstxt = temp_reqs remote = True - if requirements: + if requirementstxt: error, traceback = None, None click.echo( crayons.normal( @@ -1952,10 +1954,10 @@ def do_install( err=True, ) try: - import_requirements(r=project.path_to(requirements), dev=dev) + import_requirements(r=project.path_to(requirementstxt), dev=dev) except (UnicodeDecodeError, PipError) as e: # Don't print the temp file path if remote since it will be deleted. - req_path = requirements_url if remote else project.path_to(requirements) + req_path = requirements_url if remote else project.path_to(requirementstxt) error = ( u"Unexpected syntax in {0}. Are you sure this is a " "requirements.txt style file?".format(req_path) diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index 42248868..00aff33f 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -263,6 +263,8 @@ class Lockfile(object): """Produces a generator which generates requirements from the desired section. :param bool dev: Indicates whether to use dev requirements, defaults to False + :param bool only: When dev is True, indicates whether to use *only* dev + requirements, defaults to False :return: Requirements from the relevant the relevant pipfile :rtype: :class:`~requirementslib.models.requirements.Requirement` """ From 1e54399de7d40bd7e98eabbd5fa4b6a7f30e0d10 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 10 Apr 2020 15:58:38 +1000 Subject: [PATCH 2/7] Default dependencies, not production dependencies --- docs/advanced.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index 9fc2470b..40e33de7 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -165,7 +165,7 @@ Let's take this ``Pipfile``:: [dev-packages] pytest = {version="*"} -And generate a set of requirements out of it with only the production dependencies:: +And generate a set of requirements out of it with only the default dependencies:: $ pipenv lock -r chardet==3.0.4 @@ -174,7 +174,7 @@ And generate a set of requirements out of it with only the production dependenci idna==2.6 urllib3==1.22 -As with other commands, passing ``--dev`` will include both the production and +As with other commands, passing ``--dev`` will include both the default and development dependencies:: $ pipenv lock -r --dev From a4e69863f1908e51b22edbce3a3bb8c53b85a334 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 10 Apr 2020 16:00:20 +1000 Subject: [PATCH 3/7] Tweak NEWS wording --- news/3316.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/3316.feature.rst b/news/3316.feature.rst index 932fd759..9cab1697 100644 --- a/news/3316.feature.rst +++ b/news/3316.feature.rst @@ -1,5 +1,5 @@ For consistency with other commands and the ``-dev`` option description, ``pipenv lock --requirements --dev`` now emits both default and development dependencies. -A new ``--dev-only`` has been added to request to previous +The new ``--dev-only`` option requests the previous behaviour (e.g. to generate a ``dev-requirements.txt`` file). From b2773d608f7e4f17c9a3e5b5ceab989b9887167a Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 10 Apr 2020 16:59:39 +1000 Subject: [PATCH 4/7] Switch to new function name --- pipenv/cli/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 9c5c1a18..0c4433c6 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -409,7 +409,7 @@ def uninstall_options(f): def lock_options(f): f = install_base_options(f) - f = requirements_flag(f) + f = emit_requirements_flag(f) f = dev_only_flag(f) return f From d7842d0ab40acd1e305aa926f4e7437960aa88b2 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 11 Apr 2020 01:50:17 +1000 Subject: [PATCH 5/7] Address cosmetic parts of PEEP 6 --- pipenv/cli/command.py | 40 +++++++++++++++++++++++++++++++++------- pipenv/cli/options.py | 30 +++++++++++++++++++++++++++--- pipenv/core.py | 4 ++-- 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index c6f1e7ec..65f02af5 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -298,6 +298,19 @@ def uninstall( if retcode: sys.exit(retcode) +LOCK_HEADER = """\ +# +# These requirements were autogenerated by pipenv +# To regenerate from the project's Pipfile, run: +# +# pipenv lock {options} +# +""" + +LOCK_DEV_NOTE="""\ +# Note: in pipenv 2020.x, "--dev" changed to emit both default and development +# requirements. To emit only development requirements, pass "--dev-only". +""" @cli.command(short_help="Generates Pipfile.lock.", context_settings=CONTEXT_SETTINGS) @lock_options @@ -317,15 +330,28 @@ def lock( three=state.three, python=state.python, pypi_mirror=state.pypi_mirror, warn=(not state.quiet), site_packages=state.site_packages, ) - if state.lockoptions.emit_requirements: - # Setting "requirements=True" means do_init() just emits the + emit_requirements = state.lockoptions.emit_requirements + dev = state.installstate.dev + dev_only = state.lockoptions.dev_only + pre = state.installstate.pre + if emit_requirements: + # Emit requirements file header (unless turned off with --no-header) + if state.lockoptions.emit_requirements_header: + header_options = ["--requirements"] + if dev_only: + header_options.append("--dev-only") + elif dev: + header_options.append("--dev") + click.echo(LOCK_HEADER.format(options=" ".join(header_options))) + # TODO: Emit pip-compile style header + if dev and not dev_only: + click.echo(LOCK_DEV_NOTE) + # Setting "emit_requirements=True" means do_init() just emits the # install requirements file to stdout, it doesn't install anything - if state.installstate.dev: - pass # TODO: Emit behaviour change warning as per PEEP 006 do_init( - dev=state.installstate.dev, - dev_only=state.lockoptions.dev_only, - emit_requirements=state.lockoptions.emit_requirements, + dev=dev, + dev_only=dev_only, + emit_requirements=emit_requirements, pypi_mirror=state.pypi_mirror, pre=state.installstate.pre, ) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 0c4433c6..f958dd09 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -87,6 +87,7 @@ class LockOptions(object): def __init__(self): self.dev_only = False self.emit_requirements = False + self.emit_requirements_header = False pass_state = make_pass_decorator(State, ensure=True) @@ -172,16 +173,28 @@ def ignore_pipfile_option(f): callback=callback, type=click.types.BOOL, show_envvar=True)(f) -def dev_option(f): +def _dev_option(f, help_text): def callback(ctx, param, value): state = ctx.ensure_object(State) state.installstate.dev = value return value return option("--dev", "-d", is_flag=True, default=False, type=click.types.BOOL, - help="Install both develop and default packages.", callback=callback, + help=help_text, callback=callback, expose_value=False, show_envvar=True)(f) +def install_dev_option(f): + return _dev_option(f, "Install both develop and default packages") + + +def lock_dev_option(f): + return _dev_option(f, "Generate both develop and default requirements") + + +def uninstall_dev_option(f): + return _dev_option(f, "Uninstall both develop and default requirements") + + def pre_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) @@ -314,6 +327,15 @@ def emit_requirements_flag(f): return option("--requirements", "-r", default=False, is_flag=True, expose_value=False, help="Generate output in requirements.txt format.", callback=callback)(f) +def emit_requirements_header_flag(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + state.lockoptions.emit_requirements_header = value + return value + return option("--header/--no-header", default=True, is_flag=True, expose_value=False, + help="Add header to generated requirements", callback=callback)(f) + def dev_only_flag(f): def callback(ctx, param, value): state = ctx.ensure_object(State) @@ -393,7 +415,6 @@ def common_options(f): def install_base_options(f): f = common_options(f) - f = dev_option(f) f = pre_option(f) f = keep_outdated_option(f) return f @@ -401,6 +422,7 @@ def install_base_options(f): def uninstall_options(f): f = install_base_options(f) + f = uninstall_dev_option(f) f = skip_lock_option(f) f = editable_option(f) f = package_arg(f) @@ -409,6 +431,7 @@ def uninstall_options(f): def lock_options(f): f = install_base_options(f) + f = lock_dev_option(f) f = emit_requirements_flag(f) f = dev_only_flag(f) return f @@ -416,6 +439,7 @@ def lock_options(f): def sync_options(f): f = install_base_options(f) + f = install_dev_option(f) f = sequential_option(f) return f diff --git a/pipenv/core.py b/pipenv/core.py index 54ad73a5..f1102c0c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -791,8 +791,8 @@ def do_install_dependencies( from six.moves import queue if emit_requirements: bare = True - # Load the lockfile if it exists, or if only is being used (e.g. lock is being used). - if skip_lock or only or not project.lockfile_exists: + # Load the lockfile if it exists, or if dev_only is being used. + if skip_lock or dev_only or not project.lockfile_exists: if not bare: click.echo( crayons.normal(fix_utf8("Installing dependencies from Pipfile…"), bold=True) From 0b3c442aa13894c51bab2f277e149a97b1918f6c Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 12 Apr 2020 13:28:47 +1000 Subject: [PATCH 6/7] `--dev` has no effect on `pipenv uninstall` --- pipenv/cli/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index f958dd09..acdfe790 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -192,7 +192,7 @@ def lock_dev_option(f): def uninstall_dev_option(f): - return _dev_option(f, "Uninstall both develop and default requirements") + return _dev_option(f, "Deprecated (as it has no effect). May be removed in a future release.") def pre_option(f): From b6ca34096a04249d5f7d8d354fd0215ca8d3e25c Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 4 May 2020 16:18:03 +1000 Subject: [PATCH 7/7] Apply suggestions from code review Co-authored-by: Dan Ryan --- news/3316.feature.rst | 2 +- pipenv/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/news/3316.feature.rst b/news/3316.feature.rst index 9cab1697..63a7498a 100644 --- a/news/3316.feature.rst +++ b/news/3316.feature.rst @@ -1,4 +1,4 @@ -For consistency with other commands and the ``-dev`` option +For consistency with other commands and the ``--dev`` option description, ``pipenv lock --requirements --dev`` now emits both default and development dependencies. The new ``--dev-only`` option requests the previous diff --git a/pipenv/core.py b/pipenv/core.py index 451f6f67..5c41be35 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -808,7 +808,7 @@ def do_install_dependencies( if emit_requirements: bare = True # Load the lockfile if it exists, or if dev_only is being used. - if skip_lock or dev_only or not project.lockfile_exists: + if skip_lock or not project.lockfile_exists: if not bare: click.echo( crayons.normal(fix_utf8("Installing dependencies from Pipfile…"), bold=True)