From e818ca5af63d7480f51385b4dd3c28dab197b643 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 19:26:38 -0500 Subject: [PATCH 1/8] Fix double completion message on locking - Pass `pre` correctly when using `pipenv install --pre` - Ensures we always make inline tables when writing to pipfile - Fixes #3183 - Fixes #3185 Signed-off-by: Dan Ryan --- news/3183.bugfix.rst | 1 + news/3185.bugfix.rst | 1 + pipenv/cli/options.py | 2 +- pipenv/core.py | 34 ++++++---------- pipenv/project.py | 15 ++++--- pipenv/utils.py | 95 ++++++++++++++++++++++++++++--------------- 6 files changed, 87 insertions(+), 61 deletions(-) create mode 100644 news/3183.bugfix.rst create mode 100644 news/3185.bugfix.rst diff --git a/news/3183.bugfix.rst b/news/3183.bugfix.rst new file mode 100644 index 00000000..19e1d54a --- /dev/null +++ b/news/3183.bugfix.rst @@ -0,0 +1 @@ +Fixed new spinner success message to write only one success message during resolution. diff --git a/news/3185.bugfix.rst b/news/3185.bugfix.rst new file mode 100644 index 00000000..b6ffee2b --- /dev/null +++ b/news/3185.bugfix.rst @@ -0,0 +1 @@ +Pipenv will now correctly respect the ``--pre`` option when used with ``pipenv install``. diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 3a361c5e..208c0c66 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -338,6 +338,7 @@ def common_options(f): def install_base_options(f): f = common_options(f) f = dev_option(f) + f = pre_option(f) f = keep_outdated_option(f) return f @@ -353,7 +354,6 @@ def uninstall_options(f): def lock_options(f): f = install_base_options(f) f = requirements_flag(f) - f = pre_option(f) return f diff --git a/pipenv/core.py b/pipenv/core.py index 758ef3fa..4a292139 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1028,10 +1028,22 @@ def do_lock( deps = convert_deps_to_pip( settings["packages"], project, r=False, include_index=True ) - results = venv_resolve_deps( + # Add refs for VCS installs. + # TODO: be smarter about this. + vcs_reqs, vcs_lockfile = get_vcs_deps( + project, + which=which, + clear=clear, + pre=pre, + allow_global=system, + dev=settings["dev"], + ) + vcs_lines = [req.as_line() for req in vcs_reqs if req.editable] + results, vcs_results = venv_resolve_deps( deps, which=which, project=project, + vcs_deps=vcs_lines, clear=clear, pre=pre, allow_global=system, @@ -1045,26 +1057,6 @@ def do_lock( dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry ) lockfile[settings["lockfile_key"]].update(dep_lockfile) - # Add refs for VCS installs. - # TODO: be smarter about this. - vcs_reqs, vcs_lockfile = get_vcs_deps( - project, - which=which, - clear=clear, - pre=pre, - allow_global=system, - dev=settings["dev"], - ) - vcs_lines = [req.as_line() for req in vcs_reqs if req.editable] - vcs_results = venv_resolve_deps( - vcs_lines, - which=which, - project=project, - clear=clear, - pre=pre, - allow_global=system, - pypi_mirror=pypi_mirror, - ) for dep in vcs_results: normalized = pep423_name(dep["name"]) if not hasattr(dep, "keys") or not hasattr(dep["name"], "keys"): diff --git a/pipenv/project.py b/pipenv/project.py index 26b4cf0c..98824d94 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -617,15 +617,14 @@ class Project(object): data = tomlkit.parse(contents) # Convert all outline tables to inline tables. for section in ("packages", "dev-packages"): - table_data = data.get(section, tomlkit.table()) + table_data = data.get(section, {}) for package, value in table_data.items(): if isinstance(value, dict): - table = tomlkit.inline_table() - table.update(value) - table_data[package] = table + package_table = tomlkit.inline_table() + package_table.update(value) + data[section][package] = package_table else: - table_data[package] = value - data[section] = table_data + data[section][package] = value return data except Exception: # We lose comments here, but it's for the best.) @@ -1036,6 +1035,10 @@ class Project(object): # Skip for wildcard version return # Add the package to the group. + if isinstance(converted, dict): + package_table = tomlkit.inline_table() + package_table.update(converted) + converted = package_table p[key][name or package.normalized_name] = converted # Write Pipfile. self.write_toml(p) diff --git a/pipenv/utils.py b/pipenv/utils.py index c9feeafd..26268ab2 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -468,6 +468,37 @@ def create_spinner(text, nospin=None, spinner_name=None): yield sp +def resolve(cmd, sp): + from .vendor import delegator + from .cmdparse import Script + from .vendor.pexpect.exceptions import EOF, TIMEOUT + from .vendor.vistir.compat import to_native_string + EOF.__module__ = "pexpect.exceptions" + from ._compat import decode_output + c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) + _out = decode_output("") + result = None + out = to_native_string("") + while True: + try: + result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT) + except (EOF, TIMEOUT): + pass + if result is None: + break + _out = c.subprocess.before + if _out is not None: + _out = decode_output("{0}".format(_out)) + out += _out + sp.text = to_native_string("{0}".format(_out[:100])) + if environments.is_verbose(): + if _out is not None: + sp._hide_cursor() + sp.write(_out.rstrip()) + sp._show_cursor() + return c + + def venv_resolve_deps( deps, which, @@ -476,19 +507,16 @@ def venv_resolve_deps( clear=False, allow_global=False, pypi_mirror=None, + vcs_deps=None, ): from .vendor.vistir.misc import fs_str - from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError + from .vendor.vistir.compat import Path, JSONDecodeError from .vendor.vistir.path import create_tracked_tempdir - from .cmdparse import Script - from .vendor.pexpect.exceptions import EOF, TIMEOUT - from .vendor import delegator from . import resolver - from ._compat import decode_output import json if not deps: - return [] + return [], [] req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") cmd = [ @@ -509,29 +537,8 @@ def venv_resolve_deps( os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") - out = to_native_string("") - EOF.__module__ = "pexpect.exceptions" with create_spinner(text=fs_str("Locking...")) as sp: - c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) - _out = decode_output("") - result = None - while True: - try: - result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT) - except (EOF, TIMEOUT): - pass - if result is None: - break - _out = c.subprocess.before - if _out is not None: - _out = decode_output("{0}".format(_out)) - out += _out - sp.text = to_native_string("{0}".format(_out[:100])) - if environments.is_verbose(): - if _out is not None: - sp._hide_cursor() - sp.write(_out.rstrip()) - sp._show_cursor() + c = resolve(cmd, sp) c.block() if c.return_code != 0: sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( @@ -540,17 +547,39 @@ def venv_resolve_deps( click_echo(c.out.strip(), err=True) click_echo(c.err.strip(), err=True) sys.exit(c.return_code) + results = c.out + if vcs_deps: + with temp_environ(): + os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) + vcs_c = resolve(cmd, sp) + c.block() + if c.return_code != 0: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Locking Failed!" + )) + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) + sys.exit(c.return_code) + vcs_results = vcs_c.out + vcs_err = vcs_c.err else: - sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + vcs_results = "" + vcs_err = "" + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + outputs = [results, vcs_results] if environments.is_verbose(): - click_echo(c.out.split("RESULTS:")[0], err=True) + for output in outputs: + click_echo(output.split("RESULTS:")[0], err=True) try: - return json.loads(c.out.split("RESULTS:")[1].strip()) + results = json.loads(results.split("RESULTS:")[1].strip()) + vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) except (IndexError, JSONDecodeError): - click_echo(c.out.strip(), err=True) - click_echo(c.err.strip(), err=True) + for out, err in [(c.out, c.err), (vcs_results, vcs_err)]: + click_echo(out.strip(), err=True) + click_echo(err.strip(), err=True) raise RuntimeError("There was a problem with locking.") + return results, vcs_results def resolve_deps( From b0e5fe12f95e17341563de75ae4bb603aaeb2702 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 20:20:58 -0500 Subject: [PATCH 2/8] Fix resolver Signed-off-by: Dan Ryan --- pipenv/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 26268ab2..73f15653 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -572,7 +572,10 @@ def venv_resolve_deps( click_echo(output.split("RESULTS:")[0], err=True) try: results = json.loads(results.split("RESULTS:")[1].strip()) - vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) + if vcs_results: + vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) + else: + vcs_results = [] except (IndexError, JSONDecodeError): for out, err in [(c.out, c.err), (vcs_results, vcs_err)]: From 59961d6bbbc7ad6ce9792f792fe2d913aa41322e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 21:54:05 -0500 Subject: [PATCH 3/8] Move VCS resolution to venv and inside spinner Signed-off-by: Dan Ryan --- pipenv/core.py | 14 ++----------- pipenv/utils.py | 54 ++++++++++++++++++++++++++++--------------------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 4a292139..5316733c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1028,27 +1028,17 @@ def do_lock( deps = convert_deps_to_pip( settings["packages"], project, r=False, include_index=True ) - # Add refs for VCS installs. - # TODO: be smarter about this. - vcs_reqs, vcs_lockfile = get_vcs_deps( - project, - which=which, - clear=clear, - pre=pre, - allow_global=system, - dev=settings["dev"], - ) - vcs_lines = [req.as_line() for req in vcs_reqs if req.editable] results, vcs_results = venv_resolve_deps( deps, which=which, project=project, - vcs_deps=vcs_lines, + dev=settings["dev"], clear=clear, pre=pre, allow_global=system, pypi_mirror=pypi_mirror, ) + vcs_results, vcs_lockfile = vcs_results # Add dependencies to lockfile. for dep in results: is_top_level = dep["name"] in settings["packages"] diff --git a/pipenv/utils.py b/pipenv/utils.py index 73f15653..444333de 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -496,6 +496,14 @@ def resolve(cmd, sp): sp._hide_cursor() sp.write(_out.rstrip()) sp._show_cursor() + c.block() + if c.return_code != 0: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Locking Failed!" + )) + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) + sys.exit(c.return_code) return c @@ -507,7 +515,7 @@ def venv_resolve_deps( clear=False, allow_global=False, pypi_mirror=None, - vcs_deps=None, + dev=False, ): from .vendor.vistir.misc import fs_str from .vendor.vistir.compat import Path, JSONDecodeError @@ -515,10 +523,28 @@ def venv_resolve_deps( from . import resolver import json + vcs_deps = [] + vcs_lockfile = {} + results = [] if not deps: - return [], [] + return results, (vcs_deps, vcs_lockfile) req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") + vcs_section = "vcs_dev_packages" if dev else "vcs_packages" + if getattr(project, vcs_section, []): + with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: + vcs_reqs, vcs_lockfile = get_vcs_deps( + project, + which=which, + clear=clear, + pre=pre, + allow_global=allow_global, + dev=dev, + ) + vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] + sp.write(environments.PIPENV_SPINNER_OK_TEXT.format( + "Successfully pinned VCS Packages!" + )) cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() @@ -539,32 +565,14 @@ def venv_resolve_deps( os.environ["PIP_NO_INPUT"] = fs_str("1") with create_spinner(text=fs_str("Locking...")) as sp: c = resolve(cmd, sp) - c.block() - if c.return_code != 0: - sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( - "Locking Failed!" - )) - click_echo(c.out.strip(), err=True) - click_echo(c.err.strip(), err=True) - sys.exit(c.return_code) results = c.out if vcs_deps: with temp_environ(): os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) vcs_c = resolve(cmd, sp) - c.block() - if c.return_code != 0: - sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( - "Locking Failed!" - )) - click_echo(c.out.strip(), err=True) - click_echo(c.err.strip(), err=True) - sys.exit(c.return_code) - vcs_results = vcs_c.out - vcs_err = vcs_c.err + vcs_results, vcs_err = vcs_c.out, vcs_c.err else: - vcs_results = "" - vcs_err = "" + vcs_results, vcs_err = "", "" sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) outputs = [results, vcs_results] if environments.is_verbose(): @@ -582,7 +590,7 @@ def venv_resolve_deps( click_echo(out.strip(), err=True) click_echo(err.strip(), err=True) raise RuntimeError("There was a problem with locking.") - return results, vcs_results + return results, (vcs_results, vcs_lockfile) def resolve_deps( From 9c02f6ef5f301a62eb1574fb6ecbb819239faacb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 7 Nov 2018 22:16:14 -0500 Subject: [PATCH 4/8] Update vcs resolution Signed-off-by: Dan Ryan --- pipenv/core.py | 58 ++++++++++++++++++++++++------------------------- pipenv/utils.py | 8 ++++--- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 5316733c..ec2c1a82 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -969,7 +969,6 @@ def do_lock( pypi_mirror=None, ): """Executes the freeze functionality.""" - from .utils import get_vcs_deps cached_lockfile = {} if not pre: @@ -1931,38 +1930,39 @@ def do_install( extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) - except (ValueError, RuntimeError): + except (ValueError, RuntimeError) as e: sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) - # Warn if --editable wasn't passed. - if pkg_requirement.is_vcs and not pkg_requirement.editable: - sp.write_err( - "{0}: You installed a VCS dependency in non-editable mode. " - "This will work fine, but sub-dependencies will not be resolved by {1}." - "\n To enable this sub-dependency functionality, specify that this dependency is editable." - "".format( - crayons.red("Warning", bold=True), - crayons.red("$ pipenv lock"), - ) - ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. - if c.return_code != 0: - sp.write_err(vistir.compat.fs_str( - "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(pkg_line) - ), - )) - sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) - if "setup.py egg_info" in c.err: - sp.write_err(vistir.compat.fs_str( - "This is likely caused by a bug in {0}. " - "Report this to its maintainers.".format( - crayons.green(pkg_requirement.name) + else: + # Warn if --editable wasn't passed. + if pkg_requirement.is_vcs and not pkg_requirement.editable: + sp.write_err( + "{0}: You installed a VCS dependency in non-editable mode. " + "This will work fine, but sub-dependencies will not be resolved by {1}." + "\n To enable this sub-dependency functionality, specify that this dependency is editable." + "".format( + crayons.red("Warning", bold=True), + crayons.red("$ pipenv lock"), ) + ) + click.echo(crayons.blue(format_pip_output(c.out))) + # Ensure that package was successfully installed. + if c.return_code != 0: + sp.write_err(vistir.compat.fs_str( + "{0} An error occurred while installing {1}!".format( + crayons.red("Error: ", bold=True), crayons.green(pkg_line) + ), )) - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) - sys.exit(1) + sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) + if "setup.py egg_info" in c.err: + sp.write_err(vistir.compat.fs_str( + "This is likely caused by a bug in {0}. " + "Report this to its maintainers.".format( + crayons.green(pkg_requirement.name) + ) + )) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sys.exit(1) sp.write(vistir.compat.fs_str( u"{0} {1} {2} {3}{4}".format( crayons.normal(u"Adding", bold=True), diff --git a/pipenv/utils.py b/pipenv/utils.py index 444333de..45e37920 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -518,7 +518,7 @@ def venv_resolve_deps( dev=False, ): from .vendor.vistir.misc import fs_str - from .vendor.vistir.compat import Path, JSONDecodeError + from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError from .vendor.vistir.path import create_tracked_tempdir from . import resolver import json @@ -533,7 +533,7 @@ def venv_resolve_deps( vcs_section = "vcs_dev_packages" if dev else "vcs_packages" if getattr(project, vcs_section, []): with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: - vcs_reqs, vcs_lockfile = get_vcs_deps( + vcs_deps, vcs_lockfile = get_vcs_deps( project, which=which, clear=clear, @@ -541,7 +541,8 @@ def venv_resolve_deps( allow_global=allow_global, dev=dev, ) - vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] + vcs_deps = [req.as_line() for req in vcs_deps if req.editable] + deps.extend([req.as_line() for req in vcs_deps if not req.editable]) sp.write(environments.PIPENV_SPINNER_OK_TEXT.format( "Successfully pinned VCS Packages!" )) @@ -569,6 +570,7 @@ def venv_resolve_deps( if vcs_deps: with temp_environ(): os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) + sp.text = to_native_string("Locking VCS Dependencies...") vcs_c = resolve(cmd, sp) vcs_results, vcs_err = vcs_c.out, vcs_c.err else: From 5fbfe6a177ceab63e2a03f8bed1d363400fb7607 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 8 Nov 2018 00:27:11 -0500 Subject: [PATCH 5/8] Refactor locking Signed-off-by: Dan Ryan --- pipenv/core.py | 39 ++++++++-------------------------- pipenv/utils.py | 56 ++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 57 insertions(+), 38 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index ec2c1a82..9dbadd76 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1027,7 +1027,8 @@ def do_lock( deps = convert_deps_to_pip( settings["packages"], project, r=False, include_index=True ) - results, vcs_results = venv_resolve_deps( + lockfile_base = lockfile[settings["lockfile_key"]].copy() + locked_lockfile = venv_resolve_deps( deps, which=which, project=project, @@ -1036,33 +1037,10 @@ def do_lock( pre=pre, allow_global=system, pypi_mirror=pypi_mirror, + pipfile=settings["packages"], + lockfile=lockfile_base ) - vcs_results, vcs_lockfile = vcs_results - # Add dependencies to lockfile. - for dep in results: - is_top_level = dep["name"] in settings["packages"] - pipfile_entry = settings["packages"][dep["name"]] if is_top_level else None - dep_lockfile = clean_resolved_dep( - dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry - ) - lockfile[settings["lockfile_key"]].update(dep_lockfile) - for dep in vcs_results: - normalized = pep423_name(dep["name"]) - if not hasattr(dep, "keys") or not hasattr(dep["name"], "keys"): - continue - is_top_level = dep["name"] in vcs_lockfile or normalized in vcs_lockfile - if is_top_level: - try: - pipfile_entry = vcs_lockfile[dep["name"]] - except KeyError: - pipfile_entry = vcs_lockfile[normalized] - else: - pipfile_entry = None - dep_lockfile = clean_resolved_dep( - dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry - ) - vcs_lockfile.update(dep_lockfile) - lockfile[settings["lockfile_key"]].update(vcs_lockfile) + lockfile[settings["lockfile_key"]] = locked_lockfile # Support for --keep-outdated… if keep_outdated: @@ -1079,9 +1057,10 @@ def do_lock( section_name ][canonical_name].copy() # Overwrite any develop packages with default packages. - for default_package in lockfile["default"]: - if default_package in lockfile["develop"]: - lockfile["develop"][default_package] = lockfile["default"][default_package] + develop_keys = set(list(lockfile["develop"].keys())) + default_keys = set(list(lockfile["default"].keys())) + for pkg in default_keys & develop_keys: + lockfile["develop"][pkg] = lockfile["default"][pkg] if write: project.write_lockfile(lockfile) click.echo( diff --git a/pipenv/utils.py b/pipenv/utils.py index 45e37920..bd3e2754 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -507,6 +507,36 @@ def resolve(cmd, sp): return c +def get_locked_dep(dep, pipfile_section): + entry = None + lockfile_entry = None + if isinstance(dep, Mapping) and dep.get("name", ""): + name_options = [dep.get("name"), pep423_name(dep.get("name"))] + name = next(iter(k for k in name_options if k in pipfile_section), None) + entry = pipfile_section.get(name, None) + lockfile_entry = clean_resolved_dep(dep, is_top_level=True, pipfile_entry=entry) + else: + lockfile_entry = clean_resolved_dep(dep, is_top_level=False, pipfile_entry=entry) + return lockfile_entry + + +def prepare_lockfiles(results, pipfile, lockfile, vcs_lockfile): + from .vendor.requirementslib.utils import is_vcs + for dep in results: + # Merge in any relevant information from the pipfile entry, including + # markers, normalized names, URL info, etc that we may have dropped during lock + if not is_vcs(dep): + lockfile_entry = get_locked_dep(dep, pipfile) + lockfile.update(lockfile_entry) + # For vcs dependencies, treat the initial pass at locking (i.e. checkout) + # as the pipfile entry because it gets us an actual ref to use + else: + lockfile_entry = get_locked_dep(dep, vcs_lockfile) + vcs_lockfile.update(lockfile_entry) + lockfile.update(vcs_lockfile) + return lockfile + + def venv_resolve_deps( deps, which, @@ -516,6 +546,8 @@ def venv_resolve_deps( allow_global=False, pypi_mirror=None, dev=False, + pipfile=None, + lockfile=None ): from .vendor.vistir.misc import fs_str from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError @@ -526,14 +558,21 @@ def venv_resolve_deps( vcs_deps = [] vcs_lockfile = {} results = [] - if not deps: - return results, (vcs_deps, vcs_lockfile) + pipfile_section = "dev_packages" if dev else "packages" + lockfile_section = "develop" if dev else "default" + vcs_section = "vcs_{0}".format(pipfile_section) + vcs_deps = getattr(project, vcs_section, []) + if not deps and not vcs_deps: + return {} + if not pipfile: + pipfile = getattr(project, pipfile_section, None) + if not lockfile: + lockfile = project._lockfile[lockfile_section] req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") - vcs_section = "vcs_dev_packages" if dev else "vcs_packages" - if getattr(project, vcs_section, []): + if vcs_deps: with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: - vcs_deps, vcs_lockfile = get_vcs_deps( + vcs_reqs, vcs_lockfile = get_vcs_deps( project, which=which, clear=clear, @@ -541,8 +580,7 @@ def venv_resolve_deps( allow_global=allow_global, dev=dev, ) - vcs_deps = [req.as_line() for req in vcs_deps if req.editable] - deps.extend([req.as_line() for req in vcs_deps if not req.editable]) + vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] sp.write(environments.PIPENV_SPINNER_OK_TEXT.format( "Successfully pinned VCS Packages!" )) @@ -592,7 +630,9 @@ def venv_resolve_deps( click_echo(out.strip(), err=True) click_echo(err.strip(), err=True) raise RuntimeError("There was a problem with locking.") - return results, (vcs_results, vcs_lockfile) + results += vcs_results + lockfile = prepare_lockfiles(results, pipfile, lockfile, vcs_lockfile) + return lockfile def resolve_deps( From 3bc76bc9497978bf8314013e6bb5e3f70321915f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 8 Nov 2018 03:41:29 -0500 Subject: [PATCH 6/8] A bit more refactoring and cleanup Signed-off-by: Dan Ryan --- pipenv/core.py | 57 +++++++++++++++++++------------------------------ pipenv/utils.py | 34 ++++++++++++++--------------- 2 files changed, 39 insertions(+), 52 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 9dbadd76..b20358ef 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -959,6 +959,14 @@ def get_downloads_info(names_map, section): return info +def overwrite_dev(prod, dev): + dev_keys = set(list(dev.keys())) + prod_keys = set(list(prod.keys())) + for pkg in dev_keys & prod_keys: + dev[pkg] = prod[pkg] + return dev + + def do_lock( ctx=None, system=False, @@ -989,58 +997,40 @@ def do_lock( del lockfile[section][k] # Ensure that develop inherits from default. dev_packages = project.dev_packages.copy() - for dev_package in project.dev_packages: - if dev_package in project.packages: - dev_packages[dev_package] = project.packages[dev_package] + dev_packages = overwrite_dev(project.packages, dev_packages) # Resolve dev-package dependencies, with pip-tools. - sections = { - "dev": { - "packages": project.dev_packages, - "vcs": project.vcs_dev_packages, - "pipfile_key": "dev_packages", - "lockfile_key": "develop", - "log_string": "dev-packages", - "dev": True, - }, - "default": { - "packages": project.packages, - "vcs": project.vcs_packages, - "pipfile_key": "packages", - "lockfile_key": "default", - "log_string": "packages", - "dev": False, - }, - } - for section_name in ["dev", "default"]: - settings = sections[section_name] + for is_dev in [True, False]: + pipfile_section = "dev_packages" if is_dev else "packages" + lockfile_section = "develop" if is_dev else "default" + packages = getattr(project, pipfile_section) + if write: # Alert the user of progress. click.echo( u"{0} {1} {2}".format( crayons.normal(u"Locking"), - crayons.red(u"[{0}]".format(settings["log_string"])), + crayons.red(u"[{0}]".format(pipfile_section.replace("_", "-"))), crayons.normal(fix_utf8("dependencies…")), ), err=True, ) deps = convert_deps_to_pip( - settings["packages"], project, r=False, include_index=True + packages, project, r=False, include_index=True ) - lockfile_base = lockfile[settings["lockfile_key"]].copy() - locked_lockfile = venv_resolve_deps( + # Mutates the lockfile + venv_resolve_deps( deps, which=which, project=project, - dev=settings["dev"], + dev=is_dev, clear=clear, pre=pre, allow_global=system, pypi_mirror=pypi_mirror, - pipfile=settings["packages"], - lockfile=lockfile_base + pipfile=packages, + lockfile=lockfile ) - lockfile[settings["lockfile_key"]] = locked_lockfile # Support for --keep-outdated… if keep_outdated: @@ -1057,10 +1047,7 @@ def do_lock( section_name ][canonical_name].copy() # Overwrite any develop packages with default packages. - develop_keys = set(list(lockfile["develop"].keys())) - default_keys = set(list(lockfile["default"].keys())) - for pkg in default_keys & develop_keys: - lockfile["develop"][pkg] = lockfile["default"][pkg] + lockfile["develop"].update(overwrite_dev(lockfile.get("default", {}), lockfile["develop"])) if write: project.write_lockfile(lockfile) click.echo( diff --git a/pipenv/utils.py b/pipenv/utils.py index bd3e2754..93026ca7 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -509,18 +509,22 @@ def resolve(cmd, sp): def get_locked_dep(dep, pipfile_section): entry = None - lockfile_entry = None + cleaner_kwargs = { + "is_top_level": False, + "pipfile_entry": None + } if isinstance(dep, Mapping) and dep.get("name", ""): - name_options = [dep.get("name"), pep423_name(dep.get("name"))] + name_options = [dep["name"], pep423_name(dep["name"])] name = next(iter(k for k in name_options if k in pipfile_section), None) - entry = pipfile_section.get(name, None) - lockfile_entry = clean_resolved_dep(dep, is_top_level=True, pipfile_entry=entry) - else: - lockfile_entry = clean_resolved_dep(dep, is_top_level=False, pipfile_entry=entry) + entry = pipfile_section[name] if name else None + + if entry: + cleaner_kwargs.update({"is_top_level": True, "pipfile_entry": entry}) + lockfile_entry = clean_resolved_dep(dep, **cleaner_kwargs) return lockfile_entry -def prepare_lockfiles(results, pipfile, lockfile, vcs_lockfile): +def prepare_lockfile(results, pipfile, lockfile): from .vendor.requirementslib.utils import is_vcs for dep in results: # Merge in any relevant information from the pipfile entry, including @@ -528,12 +532,6 @@ def prepare_lockfiles(results, pipfile, lockfile, vcs_lockfile): if not is_vcs(dep): lockfile_entry = get_locked_dep(dep, pipfile) lockfile.update(lockfile_entry) - # For vcs dependencies, treat the initial pass at locking (i.e. checkout) - # as the pipfile entry because it gets us an actual ref to use - else: - lockfile_entry = get_locked_dep(dep, vcs_lockfile) - vcs_lockfile.update(lockfile_entry) - lockfile.update(vcs_lockfile) return lockfile @@ -568,7 +566,7 @@ def venv_resolve_deps( if not pipfile: pipfile = getattr(project, pipfile_section, None) if not lockfile: - lockfile = project._lockfile[lockfile_section] + lockfile = project._lockfile req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") if vcs_deps: with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: @@ -621,7 +619,10 @@ def venv_resolve_deps( try: results = json.loads(results.split("RESULTS:")[1].strip()) if vcs_results: + # For vcs dependencies, treat the initial pass at locking (i.e. checkout) + # as the pipfile entry because it gets us an actual ref to use vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) + vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile) else: vcs_results = [] @@ -630,9 +631,8 @@ def venv_resolve_deps( click_echo(out.strip(), err=True) click_echo(err.strip(), err=True) raise RuntimeError("There was a problem with locking.") - results += vcs_results - lockfile = prepare_lockfiles(results, pipfile, lockfile, vcs_lockfile) - return lockfile + lockfile[lockfile_section] = prepare_lockfile(results, pipfile, lockfile[lockfile_section]) + lockfile[lockfile_section].update(vcs_lockfile) def resolve_deps( From a0b8c6d34a5c9358ff65df6254fd79e4ff72ef18 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 8 Nov 2018 03:52:12 -0500 Subject: [PATCH 7/8] Fix syntax error Signed-off-by: Dan Ryan --- pipenv/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipenv/core.py b/pipenv/core.py index 6a492eed..cbc43d3a 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1913,6 +1913,7 @@ def do_install( crayons.red("Warning", bold=True), crayons.red("$ pipenv lock"), ) + ) click.echo(crayons.blue(format_pip_output(c.out))) # Ensure that package was successfully installed. if c.return_code != 0: From 808fd344426c526f790ccfc1f1c93c535f57da5f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 8 Nov 2018 04:49:52 -0500 Subject: [PATCH 8/8] Less checkmarks during lock Signed-off-by: Dan Ryan --- pipenv/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 93026ca7..8674aee2 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -579,9 +579,6 @@ def venv_resolve_deps( dev=dev, ) vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] - sp.write(environments.PIPENV_SPINNER_OK_TEXT.format( - "Successfully pinned VCS Packages!" - )) cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix()