Merge pull request #3190 from pypa/bugfix/3183

Fix double completion message on locking

news/3183.bugfix.rst

@@ -0,0 +1 @@
+Fixed new spinner success message to write only one success message during resolution.

news/3185.bugfix.rst

@@ -0,0 +1 @@
+Pipenv will now correctly respect the ``--pre`` option when used with ``pipenv install``.

pipenv/cli/options.py

@@ -338,6 +338,7 @@ def common_options(f):
 def install_base_options(f):
     f = common_options(f)
     f = dev_option(f)
+    f = pre_option(f)
     f = keep_outdated_option(f)
     return f
 
@@ -353,7 +354,6 @@ def uninstall_options(f):
 def lock_options(f):
     f = install_base_options(f)
     f = requirements_flag(f)
-    f = pre_option(f)
     return f
 
 

pipenv/core.py

@@ -959,6 +959,14 @@ def get_downloads_info(names_map, section):
     return info
 
 
+def overwrite_dev(prod, dev):
+    dev_keys = set(list(dev.keys()))
+    prod_keys = set(list(prod.keys()))
+    for pkg in dev_keys & prod_keys:
+        dev[pkg] = prod[pkg]
+    return dev
+
+
 def do_lock(
     ctx=None,
     system=False,
@@ -969,7 +977,6 @@ def do_lock(
     pypi_mirror=None,
 ):
     """Executes the freeze functionality."""
-    from .utils import get_vcs_deps
 
     cached_lockfile = {}
     if not pre:
@@ -990,98 +997,40 @@ def do_lock(
                 del lockfile[section][k]
     # Ensure that develop inherits from default.
     dev_packages = project.dev_packages.copy()
-    for dev_package in project.dev_packages:
-        if dev_package in project.packages:
-            dev_packages[dev_package] = project.packages[dev_package]
+    dev_packages = overwrite_dev(project.packages, dev_packages)
     # Resolve dev-package dependencies, with pip-tools.
-    sections = {
-        "dev": {
-            "packages": project.dev_packages,
-            "vcs": project.vcs_dev_packages,
-            "pipfile_key": "dev_packages",
-            "lockfile_key": "develop",
-            "log_string": "dev-packages",
-            "dev": True,
-        },
-        "default": {
-            "packages": project.packages,
-            "vcs": project.vcs_packages,
-            "pipfile_key": "packages",
-            "lockfile_key": "default",
-            "log_string": "packages",
-            "dev": False,
-        },
-    }
-    for section_name in ["dev", "default"]:
-        settings = sections[section_name]
+    for is_dev in [True, False]:
+        pipfile_section = "dev_packages" if is_dev else "packages"
+        lockfile_section = "develop" if is_dev else "default"
+        packages = getattr(project, pipfile_section)
+
         if write:
             # Alert the user of progress.
             click.echo(
                 u"{0} {1} {2}".format(
                     crayons.normal(u"Locking"),
-                    crayons.red(u"[{0}]".format(settings["log_string"])),
+                    crayons.red(u"[{0}]".format(pipfile_section.replace("_", "-"))),
                     crayons.normal(fix_utf8("dependencies…")),
                 ),
                 err=True,
             )
 
         deps = convert_deps_to_pip(
-            settings["packages"], project, r=False, include_index=True
+            packages, project, r=False, include_index=True
         )
-        results = venv_resolve_deps(
+        # Mutates the lockfile
+        venv_resolve_deps(
             deps,
             which=which,
             project=project,
+            dev=is_dev,
             clear=clear,
             pre=pre,
             allow_global=system,
             pypi_mirror=pypi_mirror,
+            pipfile=packages,
+            lockfile=lockfile
         )
-        # Add dependencies to lockfile.
-        for dep in results:
-            is_top_level = dep["name"] in settings["packages"]
-            pipfile_entry = settings["packages"][dep["name"]] if is_top_level else None
-            dep_lockfile = clean_resolved_dep(
-                dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry
-            )
-            lockfile[settings["lockfile_key"]].update(dep_lockfile)
-        # Add refs for VCS installs.
-        # TODO: be smarter about this.
-        vcs_reqs, vcs_lockfile = get_vcs_deps(
-            project,
-            which=which,
-            clear=clear,
-            pre=pre,
-            allow_global=system,
-            dev=settings["dev"],
-        )
-        vcs_lines = [req.as_line() for req in vcs_reqs if req.editable]
-        vcs_results = venv_resolve_deps(
-            vcs_lines,
-            which=which,
-            project=project,
-            clear=clear,
-            pre=pre,
-            allow_global=system,
-            pypi_mirror=pypi_mirror,
-        )
-        for dep in vcs_results:
-            normalized = pep423_name(dep["name"])
-            if not hasattr(dep, "keys") or not hasattr(dep["name"], "keys"):
-                continue
-            is_top_level = dep["name"] in vcs_lockfile or normalized in vcs_lockfile
-            if is_top_level:
-                try:
-                    pipfile_entry = vcs_lockfile[dep["name"]]
-                except KeyError:
-                    pipfile_entry = vcs_lockfile[normalized]
-            else:
-                pipfile_entry = None
-            dep_lockfile = clean_resolved_dep(
-                dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry
-            )
-            vcs_lockfile.update(dep_lockfile)
-        lockfile[settings["lockfile_key"]].update(vcs_lockfile)
 
     # Support for --keep-outdated…
     if keep_outdated:
@@ -1098,9 +1047,7 @@ def do_lock(
                             section_name
                         ][canonical_name].copy()
     # Overwrite any develop packages with default packages.
-    for default_package in lockfile["default"]:
-        if default_package in lockfile["develop"]:
-            lockfile["develop"][default_package] = lockfile["default"][default_package]
+    lockfile["develop"].update(overwrite_dev(lockfile.get("default", {}), lockfile["develop"]))
     if write:
         project.write_lockfile(lockfile)
         click.echo(
@@ -1967,24 +1914,24 @@ def do_install(
                             crayons.red("$ pipenv lock"),
                         )
                     )
-                click.echo(crayons.blue(format_pip_output(c.out)))
-                # Ensure that package was successfully installed.
-                if c.return_code != 0:
-                    sp.write_err(vistir.compat.fs_str(
-                        "{0} An error occurred while installing {1}!".format(
-                            crayons.red("Error: ", bold=True), crayons.green(pkg_line)
-                        ),
-                    ))
-                    sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err))))
-                    if "setup.py egg_info" in c.err:
+                    click.echo(crayons.blue(format_pip_output(c.out)))
+                    # Ensure that package was successfully installed.
+                    if c.return_code != 0:
                         sp.write_err(vistir.compat.fs_str(
-                            "This is likely caused by a bug in {0}. "
-                            "Report this to its maintainers.".format(
-                                crayons.green(pkg_requirement.name)
-                            )
+                            "{0} An error occurred while installing {1}!".format(
+                                crayons.red("Error: ", bold=True), crayons.green(pkg_line)
+                            ),
                         ))
-                    sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed"))
-                    sys.exit(1)
+                        sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err))))
+                        if "setup.py egg_info" in c.err:
+                            sp.write_err(vistir.compat.fs_str(
+                                "This is likely caused by a bug in {0}. "
+                                "Report this to its maintainers.".format(
+                                    crayons.green(pkg_requirement.name)
+                                )
+                            ))
+                        sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed"))
+                        sys.exit(1)
                 sp.write(vistir.compat.fs_str(
                     u"{0} {1} {2} {3}{4}".format(
                         crayons.normal(u"Adding", bold=True),

pipenv/project.py

@@ -617,15 +617,14 @@ class Project(object):
             data = tomlkit.parse(contents)
             # Convert all outline tables to inline tables.
             for section in ("packages", "dev-packages"):
-                table_data = data.get(section, tomlkit.table())
+                table_data = data.get(section, {})
                 for package, value in table_data.items():
                     if isinstance(value, dict):
-                        table = tomlkit.inline_table()
-                        table.update(value)
-                        table_data[package] = table
+                        package_table = tomlkit.inline_table()
+                        package_table.update(value)
+                        data[section][package] = package_table
                     else:
-                        table_data[package] = value
-                data[section] = table_data
+                        data[section][package] = value
             return data
         except Exception:
             # We lose comments here, but it's for the best.)
@@ -1036,6 +1035,10 @@ class Project(object):
             # Skip for wildcard version
             return
         # Add the package to the group.
+        if isinstance(converted, dict):
+            package_table = tomlkit.inline_table()
+            package_table.update(converted)
+            converted = package_table
         p[key][name or package.normalized_name] = converted
         # Write Pipfile.
         self.write_toml(p)

pipenv/utils.py

@@ -468,6 +468,73 @@ def create_spinner(text, nospin=None, spinner_name=None):
         yield sp
 
 
+def resolve(cmd, sp):
+    from .vendor import delegator
+    from .cmdparse import Script
+    from .vendor.pexpect.exceptions import EOF, TIMEOUT
+    from .vendor.vistir.compat import to_native_string
+    EOF.__module__ = "pexpect.exceptions"
+    from ._compat import decode_output
+    c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy())
+    _out = decode_output("")
+    result = None
+    out = to_native_string("")
+    while True:
+        try:
+            result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT)
+        except (EOF, TIMEOUT):
+            pass
+        if result is None:
+            break
+        _out = c.subprocess.before
+        if _out is not None:
+            _out = decode_output("{0}".format(_out))
+            out += _out
+            sp.text = to_native_string("{0}".format(_out[:100]))
+        if environments.is_verbose():
+            if _out is not None:
+                sp._hide_cursor()
+                sp.write(_out.rstrip())
+                sp._show_cursor()
+    c.block()
+    if c.return_code != 0:
+        sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format(
+            "Locking Failed!"
+        ))
+        click_echo(c.out.strip(), err=True)
+        click_echo(c.err.strip(), err=True)
+        sys.exit(c.return_code)
+    return c
+
+
+def get_locked_dep(dep, pipfile_section):
+    entry = None
+    cleaner_kwargs = {
+        "is_top_level": False,
+        "pipfile_entry": None
+    }
+    if isinstance(dep, Mapping) and dep.get("name", ""):
+        name_options = [dep["name"], pep423_name(dep["name"])]
+        name = next(iter(k for k in name_options if k in pipfile_section), None)
+        entry = pipfile_section[name] if name else None
+
+    if entry:
+        cleaner_kwargs.update({"is_top_level": True, "pipfile_entry": entry})
+    lockfile_entry = clean_resolved_dep(dep, **cleaner_kwargs)
+    return lockfile_entry
+
+
+def prepare_lockfile(results, pipfile, lockfile):
+    from .vendor.requirementslib.utils import is_vcs
+    for dep in results:
+        # Merge in any relevant information from the pipfile entry, including
+        # markers, normalized names, URL info, etc that we may have dropped during lock
+        if not is_vcs(dep):
+            lockfile_entry = get_locked_dep(dep, pipfile)
+            lockfile.update(lockfile_entry)
+    return lockfile
+
+
 def venv_resolve_deps(
     deps,
     which,
@@ -476,21 +543,42 @@ def venv_resolve_deps(
     clear=False,
     allow_global=False,
     pypi_mirror=None,
+    dev=False,
+    pipfile=None,
+    lockfile=None
 ):
     from .vendor.vistir.misc import fs_str
     from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError
     from .vendor.vistir.path import create_tracked_tempdir
-    from .cmdparse import Script
-    from .vendor.pexpect.exceptions import EOF, TIMEOUT
-    from .vendor import delegator
     from . import resolver
-    from ._compat import decode_output
     import json
 
-    if not deps:
-        return []
+    vcs_deps = []
+    vcs_lockfile = {}
+    results = []
+    pipfile_section = "dev_packages" if dev else "packages"
+    lockfile_section = "develop" if dev else "default"
+    vcs_section = "vcs_{0}".format(pipfile_section)
+    vcs_deps = getattr(project, vcs_section, [])
+    if not deps and not vcs_deps:
+        return {}
 
+    if not pipfile:
+        pipfile = getattr(project, pipfile_section, None)
+    if not lockfile:
+        lockfile = project._lockfile
     req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements")
+    if vcs_deps:
+        with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp:
+            vcs_reqs, vcs_lockfile = get_vcs_deps(
+                project,
+                which=which,
+                clear=clear,
+                pre=pre,
+                allow_global=allow_global,
+                dev=dev,
+            )
+            vcs_deps = [req.as_line() for req in vcs_reqs if req.editable]
     cmd = [
         which("python", allow_global=allow_global),
         Path(resolver.__file__.rstrip("co")).as_posix()
@@ -509,48 +597,39 @@ def venv_resolve_deps(
         os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY)
         os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir)
         os.environ["PIP_NO_INPUT"] = fs_str("1")
-        out = to_native_string("")
-        EOF.__module__ = "pexpect.exceptions"
         with create_spinner(text=fs_str("Locking...")) as sp:
-            c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy())
-            _out = decode_output("")
-            result = None
-            while True:
-                try:
-                    result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT)
-                except (EOF, TIMEOUT):
-                    pass
-                if result is None:
-                    break
-                _out = c.subprocess.before
-                if _out is not None:
-                    _out = decode_output("{0}".format(_out))
-                    out += _out
-                    sp.text = to_native_string("{0}".format(_out[:100]))
-                if environments.is_verbose():
-                    if _out is not None:
-                        sp._hide_cursor()
-                        sp.write(_out.rstrip())
-                        sp._show_cursor()
-            c.block()
-            if c.return_code != 0:
-                sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format(
-                    "Locking Failed!"
-                ))
-                click_echo(c.out.strip(), err=True)
-                click_echo(c.err.strip(), err=True)
-                sys.exit(c.return_code)
+            c = resolve(cmd, sp)
+            results = c.out
+            if vcs_deps:
+                with temp_environ():
+                    os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps))
+                    sp.text = to_native_string("Locking VCS Dependencies...")
+                    vcs_c = resolve(cmd, sp)
+                    vcs_results, vcs_err = vcs_c.out, vcs_c.err
             else:
-                sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!"))
+                vcs_results, vcs_err = "", ""
+            sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!"))
+    outputs = [results, vcs_results]
     if environments.is_verbose():
-        click_echo(c.out.split("RESULTS:")[0], err=True)
+        for output in outputs:
+            click_echo(output.split("RESULTS:")[0], err=True)
     try:
-        return json.loads(c.out.split("RESULTS:")[1].strip())
+        results = json.loads(results.split("RESULTS:")[1].strip())
+        if vcs_results:
+            # For vcs dependencies, treat the initial pass at locking (i.e. checkout)
+            # as the pipfile entry because it gets us an actual ref to use
+            vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip())
+            vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile)
+        else:
+            vcs_results = []
 
     except (IndexError, JSONDecodeError):
-        click_echo(c.out.strip(), err=True)
-        click_echo(c.err.strip(), err=True)
+        for out, err in [(c.out, c.err), (vcs_results, vcs_err)]:
+            click_echo(out.strip(), err=True)
+            click_echo(err.strip(), err=True)
         raise RuntimeError("There was a problem with locking.")
+    lockfile[lockfile_section] = prepare_lockfile(results, pipfile, lockfile[lockfile_section])
+    lockfile[lockfile_section].update(vcs_lockfile)
 
 
 def resolve_deps(