Merge pull request #4183 from ncoghlan/issue-3316-lock-all-deps-with-dev

docs/advanced.rst

@@ -149,7 +149,9 @@ Anaconda uses Conda to manage packages. To reuse Conda–installed Python packag
 ☤ Generating a ``requirements.txt``
 -----------------------------------
 
-You can convert a ``Pipfile`` and ``Pipfile.lock`` into a ``requirements.txt`` file very easily, and get all the benefits of extras and other goodies we have included.
+You can convert a ``Pipfile`` and ``Pipfile.lock`` into a ``requirements.txt``
+file very easily, and get all the benefits of extras and other goodies we have
+included.
 
 Let's take this ``Pipfile``::
 
@@ -160,7 +162,10 @@ Let's take this ``Pipfile``::
     [packages]
     requests = {version="*"}
 
-And generate a ``requirements.txt`` out of it::
+    [dev-packages]
+    pytest = {version="*"}
+
+And generate a set of requirements out of it with only the default dependencies::
 
     $ pipenv lock -r
     chardet==3.0.4
@@ -169,22 +174,41 @@ And generate a ``requirements.txt`` out of it::
     idna==2.6
     urllib3==1.22
 
-If you wish to generate a ``requirements.txt`` with only the development requirements you can do that too!  Let's take the following ``Pipfile``::
-
-    [[source]]
-    url = "https://pypi.python.org/simple"
-    verify_ssl = true
-
-    [dev-packages]
-    pytest = {version="*"}
-
-And generate a ``requirements.txt`` out of it::
+As with other commands, passing ``--dev`` will include both the default and
+development dependencies::
 
     $ pipenv lock -r --dev
+    chardet==3.0.4
+    requests==2.18.4
+    certifi==2017.7.27.1
+    idna==2.6
+    urllib3==1.22
+    py==1.4.34
+    pytest==3.2.3
+
+Finally, if you wish to generate a requirements file with only the
+development requirements you can do that too, using the ``--dev-only``
+flag::
+
+    $ pipenv lock -r --dev-only
+    py==1.4.34
+    pytest==3.2.3
+
+The locked requirements are written to stdout, with shell output redirection
+used to write them to a file::
+
+    $ pipenv lock -r > requirements.txt
+    $ pipenv lock -r --dev-only > dev-requirements.txt
+    $ cat requirements.txt
+    chardet==3.0.4
+    requests==2.18.4
+    certifi==2017.7.27.1
+    idna==2.6
+    urllib3==1.22
+    $ cat dev-requirements.txt
     py==1.4.34
     pytest==3.2.3
 
-Very fancy.
 
 ☤ Detection of Security Vulnerabilities
 ---------------------------------------

news/3316.feature.rst

@@ -0,0 +1,5 @@
+For consistency with other commands and the ``--dev`` option
+description, ``pipenv lock --requirements --dev`` now emits
+both default and development dependencies.
+The new ``--dev-only`` option requests the previous
+behaviour (e.g. to generate a ``dev-requirements.txt`` file).

pipenv/cli/command.py

@@ -237,7 +237,7 @@ def install(
         lock=not state.installstate.skip_lock,
         ignore_pipfile=state.installstate.ignore_pipfile,
         skip_lock=state.installstate.skip_lock,
-        requirements=state.installstate.requirementstxt,
+        requirementstxt=state.installstate.requirementstxt,
         sequential=state.installstate.sequential,
         pre=state.installstate.pre,
         code=state.installstate.code,
@@ -298,6 +298,19 @@ def uninstall(
     if retcode:
         sys.exit(retcode)
 
+LOCK_HEADER = """\
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock {options}
+#
+"""
+
+LOCK_DEV_NOTE="""\
+# Note: in pipenv 2020.x, "--dev" changed to emit both default and development
+# requirements. To emit only development requirements, pass "--dev-only".
+"""
 
 @cli.command(short_help="Generates Pipfile.lock.", context_settings=CONTEXT_SETTINGS)
 @lock_options
@@ -317,13 +330,37 @@ def lock(
         three=state.three, python=state.python, pypi_mirror=state.pypi_mirror,
         warn=(not state.quiet), site_packages=state.site_packages,
     )
-    if state.installstate.requirementstxt:
+    emit_requirements = state.lockoptions.emit_requirements
+    dev = state.installstate.dev
+    dev_only = state.lockoptions.dev_only
+    pre = state.installstate.pre
+    if emit_requirements:
+        # Emit requirements file header (unless turned off with --no-header)
+        if state.lockoptions.emit_requirements_header:
+            header_options = ["--requirements"]
+            if dev_only:
+                header_options.append("--dev-only")
+            elif dev:
+                header_options.append("--dev")
+            click.echo(LOCK_HEADER.format(options=" ".join(header_options)))
+            # TODO: Emit pip-compile style header
+            if dev and not dev_only:
+                click.echo(LOCK_DEV_NOTE)
+        # Setting "emit_requirements=True" means do_init() just emits the
+        # install requirements file to stdout, it doesn't install anything
         do_init(
-            dev=state.installstate.dev,
-            requirements=state.installstate.requirementstxt,
+            dev=dev,
+            dev_only=dev_only,
+            emit_requirements=emit_requirements,
             pypi_mirror=state.pypi_mirror,
             pre=state.installstate.pre,
         )
+    elif state.lockoptions.dev_only:
+        raise exceptions.PipenvOptionsError(
+            "--dev-only",
+            "--dev-only is only permitted in combination with --requirements. "
+            "Aborting."
+        )
     do_lock(
         ctx=ctx,
         clear=state.clear,

pipenv/cli/options.py

@@ -65,6 +65,7 @@ class State(object):
         self.clear = False
         self.system = False
         self.installstate = InstallState()
+        self.lockoptions = LockOptions()
 
 
 class InstallState(object):
@@ -82,6 +83,11 @@ class InstallState(object):
         self.packages = []
         self.editables = []
 
+class LockOptions(object):
+    def __init__(self):
+        self.dev_only = False
+        self.emit_requirements = False
+        self.emit_requirements_header = False
 
 pass_state = make_pass_decorator(State, ensure=True)
 
@@ -169,16 +175,28 @@ def ignore_pipfile_option(f):
                   callback=callback, type=click.types.BOOL, show_envvar=True)(f)
 
 
-def dev_option(f):
+def _dev_option(f, help_text):
     def callback(ctx, param, value):
         state = ctx.ensure_object(State)
         state.installstate.dev = value
         return value
     return option("--dev", "-d", is_flag=True, default=False, type=click.types.BOOL,
-                  help="Install both develop and default packages.", callback=callback,
+                  help=help_text, callback=callback,
                   expose_value=False, show_envvar=True)(f)
 
 
+def install_dev_option(f):
+    return _dev_option(f, "Install both develop and default packages")
+
+
+def lock_dev_option(f):
+    return _dev_option(f, "Generate both develop and default requirements")
+
+
+def uninstall_dev_option(f):
+    return _dev_option(f, "Deprecated (as it has no effect). May be removed in a future release.")
+
+
 def pre_option(f):
     def callback(ctx, param, value):
         state = ctx.ensure_object(State)
@@ -302,15 +320,32 @@ def requirementstxt_option(f):
                   help="Import a requirements.txt file.", callback=callback)(f)
 
 
-def requirements_flag(f):
+def emit_requirements_flag(f):
     def callback(ctx, param, value):
         state = ctx.ensure_object(State)
         if value:
-            state.installstate.requirementstxt = value
+            state.lockoptions.emit_requirements = value
         return value
     return option("--requirements", "-r", default=False, is_flag=True, expose_value=False,
                   help="Generate output in requirements.txt format.", callback=callback)(f)
 
+def emit_requirements_header_flag(f):
+    def callback(ctx, param, value):
+        state = ctx.ensure_object(State)
+        if value:
+            state.lockoptions.emit_requirements_header = value
+        return value
+    return option("--header/--no-header", default=True, is_flag=True, expose_value=False,
+                  help="Add header to generated requirements", callback=callback)(f)
+
+def dev_only_flag(f):
+    def callback(ctx, param, value):
+        state = ctx.ensure_object(State)
+        if value:
+            state.lockoptions.dev_only = value
+        return value
+    return option("--dev-only", default=False, is_flag=True, expose_value=False,
+                  help="Emit development dependencies *only* (overrides --dev)", callback=callback)(f)
 
 def code_option(f):
     def callback(ctx, param, value):
@@ -382,7 +417,6 @@ def common_options(f):
 
 def install_base_options(f):
     f = common_options(f)
-    f = dev_option(f)
     f = pre_option(f)
     f = keep_outdated_option(f)
     return f
@@ -390,6 +424,7 @@ def install_base_options(f):
 
 def uninstall_options(f):
     f = install_base_options(f)
+    f = uninstall_dev_option(f)
     f = skip_lock_option(f)
     f = editable_option(f)
     f = package_arg(f)
@@ -398,12 +433,15 @@ def uninstall_options(f):
 
 def lock_options(f):
     f = install_base_options(f)
-    f = requirements_flag(f)
+    f = lock_dev_option(f)
+    f = emit_requirements_flag(f)
+    f = dev_only_flag(f)
     return f
 
 
 def sync_options(f):
     f = install_base_options(f)
+    f = install_dev_option(f)
     f = sequential_option(f)
     return f
 

pipenv/core.py

@@ -802,9 +802,9 @@ def batch_install(deps_list, procs, failed_deps_queue,
 
 def do_install_dependencies(
     dev=False,
-    only=False,
+    dev_only=False,
     bare=False,
-    requirements=False,
+    emit_requirements=False,
     allow_global=False,
     ignore_hashes=False,
     skip_lock=False,
@@ -815,14 +815,14 @@ def do_install_dependencies(
     """"
     Executes the install functionality.
 
-    If requirements is True, simply spits out a requirements format to stdout.
+    If emit_requirements is True, simply spits out a requirements format to stdout.
     """
 
     from six.moves import queue
-    if requirements:
+    if emit_requirements:
         bare = True
-    # Load the lockfile if it exists, or if only is being used (e.g. lock is being used).
-    if skip_lock or only or not project.lockfile_exists:
+    # Load the lockfile if it exists, or if dev_only is being used.
+    if skip_lock or not project.lockfile_exists:
         if not bare:
             click.echo(
                 crayons.normal(fix_utf8("Installing dependencies from Pipfile…"), bold=True)
@@ -842,14 +842,14 @@ def do_install_dependencies(
             )
     # Allow pip to resolve dependencies when in skip-lock mode.
     no_deps = not skip_lock  # skip_lock true, no_deps False, pip resolves deps
-    deps_list = list(lockfile.get_requirements(dev=dev, only=requirements))
-    if requirements:
+    dev = dev or dev_only
+    deps_list = list(lockfile.get_requirements(dev=dev, only=dev_only))
+    if emit_requirements:
         index_args = prepare_pip_source_args(get_source_list(pypi_mirror=pypi_mirror, project=project))
         index_args = " ".join(index_args).replace(" -", "\n-")
         deps = [
             req.as_line(sources=False, include_hashes=False) for req in deps_list
         ]
-        # Output only default dependencies
         click.echo(index_args)
         click.echo(
             "\n".join(sorted(deps))
@@ -1201,7 +1201,8 @@ def do_purge(bare=False, downloads=False, allow_global=False):
 
 def do_init(
     dev=False,
-    requirements=False,
+    dev_only=False,
+    emit_requirements=False,
     allow_global=False,
     ignore_pipfile=False,
     skip_lock=False,
@@ -1306,7 +1307,8 @@ def do_init(
             )
     do_install_dependencies(
         dev=dev,
-        requirements=requirements,
+        dev_only=dev_only,
+        emit_requirements=emit_requirements,
         allow_global=allow_global,
         skip_lock=skip_lock,
         concurrent=concurrent,
@@ -1891,7 +1893,7 @@ def do_install(
     lock=True,
     ignore_pipfile=False,
     skip_lock=False,
-    requirements=False,
+    requirementstxt=False,
     sequential=False,
     pre=False,
     code=False,
@@ -1914,7 +1916,7 @@ def do_install(
     package_args = [p for p in packages if p] + [p for p in editable_packages if p]
     skip_requirements = False
     # Don't search for requirements.txt files if the user provides one
-    if requirements or package_args or project.pipfile_exists:
+    if requirementstxt or package_args or project.pipfile_exists:
         skip_requirements = True
     concurrent = not sequential
     # Ensure that virtualenv is available and pipfile are available
@@ -1939,7 +1941,7 @@ def do_install(
         pre = project.settings.get("allow_prereleases")
     if not keep_outdated:
         keep_outdated = project.settings.get("keep_outdated")
-    remote = requirements and is_valid_url(requirements)
+    remote = requirementstxt and is_valid_url(requirementstxt)
     # Warn and exit if --system is used without a pipfile.
     if (system and package_args) and not (PIPENV_VIRTUALENV):
         raise exceptions.SystemUsageError
@@ -1958,17 +1960,17 @@ def do_install(
             prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory
         )
         temp_reqs = fd.name
-        requirements_url = requirements
+        requirements_url = requirementstxt
         # Download requirements file
         try:
-            download_file(requirements, temp_reqs)
+            download_file(requirements_url, temp_reqs)
         except IOError:
             fd.close()
             os.unlink(temp_reqs)
             click.echo(
                 crayons.red(
                     u"Unable to find requirements file at {0}.".format(
-                        crayons.normal(requirements)
+                        crayons.normal(requirements_url)
                     )
                 ),
                 err=True,
@@ -1977,9 +1979,9 @@ def do_install(
         finally:
             fd.close()
         # Replace the url with the temporary requirements file
-        requirements = temp_reqs
+        requirementstxt = temp_reqs
         remote = True
-    if requirements:
+    if requirementstxt:
         error, traceback = None, None
         click.echo(
             crayons.normal(
@@ -1988,10 +1990,10 @@ def do_install(
             err=True,
         )
         try:
-            import_requirements(r=project.path_to(requirements), dev=dev)
+            import_requirements(r=project.path_to(requirementstxt), dev=dev)
         except (UnicodeDecodeError, PipError) as e:
             # Don't print the temp file path if remote since it will be deleted.
-            req_path = requirements_url if remote else project.path_to(requirements)
+            req_path = requirements_url if remote else project.path_to(requirementstxt)
             error = (
                 u"Unexpected syntax in {0}. Are you sure this is a "
                 "requirements.txt style file?".format(req_path)

pipenv/vendor/requirementslib/models/lockfile.py

@@ -264,6 +264,8 @@ class Lockfile(object):
         """Produces a generator which generates requirements from the desired section.
 
         :param bool dev: Indicates whether to use dev requirements, defaults to False
+        :param bool only: When dev is True, indicates whether to use *only* dev
+                          requirements, defaults to False
         :return: Requirements from the relevant the relevant pipfile
         :rtype: :class:`~requirementslib.models.requirements.Requirement`
         """