diff --git a/news/3183.bugfix.rst b/news/3183.bugfix.rst new file mode 100644 index 00000000..19e1d54a --- /dev/null +++ b/news/3183.bugfix.rst @@ -0,0 +1 @@ +Fixed new spinner success message to write only one success message during resolution. diff --git a/news/3185.bugfix.rst b/news/3185.bugfix.rst new file mode 100644 index 00000000..b6ffee2b --- /dev/null +++ b/news/3185.bugfix.rst @@ -0,0 +1 @@ +Pipenv will now correctly respect the ``--pre`` option when used with ``pipenv install``. diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 3a361c5e..208c0c66 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -338,6 +338,7 @@ def common_options(f): def install_base_options(f): f = common_options(f) f = dev_option(f) + f = pre_option(f) f = keep_outdated_option(f) return f @@ -353,7 +354,6 @@ def uninstall_options(f): def lock_options(f): f = install_base_options(f) f = requirements_flag(f) - f = pre_option(f) return f diff --git a/pipenv/core.py b/pipenv/core.py index f9b8eaee..dabf065c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -454,8 +454,8 @@ def ensure_python(three=None, python=None): click.echo(fix_utf8("Something went wrong…"), err=True) click.echo(crayons.blue(e.err), err=True) else: - environments.PIPENV_SPINNER_OK_TEXT.format("Success!") - # Print the results, in a beautiful blue… + sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + # Print the results, in a beautiful blue… click.echo(crayons.blue(c.out), err=True) # Find the newly installed Python, hopefully. version = str(version) @@ -959,6 +959,14 @@ def get_downloads_info(names_map, section): return info +def overwrite_dev(prod, dev): + dev_keys = set(list(dev.keys())) + prod_keys = set(list(prod.keys())) + for pkg in dev_keys & prod_keys: + dev[pkg] = prod[pkg] + return dev + + def do_lock( ctx=None, system=False, @@ -969,7 +977,6 @@ def do_lock( pypi_mirror=None, ): """Executes the freeze functionality.""" - from .utils import get_vcs_deps cached_lockfile = {} if not pre: @@ -990,98 +997,40 @@ def do_lock( del lockfile[section][k] # Ensure that develop inherits from default. dev_packages = project.dev_packages.copy() - for dev_package in project.dev_packages: - if dev_package in project.packages: - dev_packages[dev_package] = project.packages[dev_package] + dev_packages = overwrite_dev(project.packages, dev_packages) # Resolve dev-package dependencies, with pip-tools. - sections = { - "dev": { - "packages": project.dev_packages, - "vcs": project.vcs_dev_packages, - "pipfile_key": "dev_packages", - "lockfile_key": "develop", - "log_string": "dev-packages", - "dev": True, - }, - "default": { - "packages": project.packages, - "vcs": project.vcs_packages, - "pipfile_key": "packages", - "lockfile_key": "default", - "log_string": "packages", - "dev": False, - }, - } - for section_name in ["dev", "default"]: - settings = sections[section_name] + for is_dev in [True, False]: + pipfile_section = "dev_packages" if is_dev else "packages" + lockfile_section = "develop" if is_dev else "default" + packages = getattr(project, pipfile_section) + if write: # Alert the user of progress. click.echo( u"{0} {1} {2}".format( crayons.normal(u"Locking"), - crayons.red(u"[{0}]".format(settings["log_string"])), + crayons.red(u"[{0}]".format(pipfile_section.replace("_", "-"))), crayons.normal(fix_utf8("dependencies…")), ), err=True, ) deps = convert_deps_to_pip( - settings["packages"], project, r=False, include_index=True + packages, project, r=False, include_index=True ) - results = venv_resolve_deps( + # Mutates the lockfile + venv_resolve_deps( deps, which=which, project=project, + dev=is_dev, clear=clear, pre=pre, allow_global=system, pypi_mirror=pypi_mirror, + pipfile=packages, + lockfile=lockfile ) - # Add dependencies to lockfile. - for dep in results: - is_top_level = dep["name"] in settings["packages"] - pipfile_entry = settings["packages"][dep["name"]] if is_top_level else None - dep_lockfile = clean_resolved_dep( - dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry - ) - lockfile[settings["lockfile_key"]].update(dep_lockfile) - # Add refs for VCS installs. - # TODO: be smarter about this. - vcs_reqs, vcs_lockfile = get_vcs_deps( - project, - which=which, - clear=clear, - pre=pre, - allow_global=system, - dev=settings["dev"], - ) - vcs_lines = [req.as_line() for req in vcs_reqs if req.editable] - vcs_results = venv_resolve_deps( - vcs_lines, - which=which, - project=project, - clear=clear, - pre=pre, - allow_global=system, - pypi_mirror=pypi_mirror, - ) - for dep in vcs_results: - normalized = pep423_name(dep["name"]) - if not hasattr(dep, "keys") or not hasattr(dep["name"], "keys"): - continue - is_top_level = dep["name"] in vcs_lockfile or normalized in vcs_lockfile - if is_top_level: - try: - pipfile_entry = vcs_lockfile[dep["name"]] - except KeyError: - pipfile_entry = vcs_lockfile[normalized] - else: - pipfile_entry = None - dep_lockfile = clean_resolved_dep( - dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry - ) - vcs_lockfile.update(dep_lockfile) - lockfile[settings["lockfile_key"]].update(vcs_lockfile) # Support for --keep-outdated… if keep_outdated: @@ -1098,9 +1047,7 @@ def do_lock( section_name ][canonical_name].copy() # Overwrite any develop packages with default packages. - for default_package in lockfile["default"]: - if default_package in lockfile["develop"]: - lockfile["develop"][default_package] = lockfile["default"][default_package] + lockfile["develop"].update(overwrite_dev(lockfile.get("default", {}), lockfile["develop"])) if write: project.write_lockfile(lockfile) click.echo( @@ -1967,24 +1914,24 @@ def do_install( crayons.red("$ pipenv lock"), ) ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. - if c.return_code != 0: - sp.write_err(vistir.compat.fs_str( - "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(pkg_line) - ), - )) - sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) - if "setup.py egg_info" in c.err: + click.echo(crayons.blue(format_pip_output(c.out))) + # Ensure that package was successfully installed. + if c.return_code != 0: sp.write_err(vistir.compat.fs_str( - "This is likely caused by a bug in {0}. " - "Report this to its maintainers.".format( - crayons.green(pkg_requirement.name) - ) + "{0} An error occurred while installing {1}!".format( + crayons.red("Error: ", bold=True), crayons.green(pkg_line) + ), )) - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) - sys.exit(1) + sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) + if "setup.py egg_info" in c.err: + sp.write_err(vistir.compat.fs_str( + "This is likely caused by a bug in {0}. " + "Report this to its maintainers.".format( + crayons.green(pkg_requirement.name) + ) + )) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sys.exit(1) sp.write(vistir.compat.fs_str( u"{0} {1} {2} {3}{4}".format( crayons.normal(u"Adding", bold=True), diff --git a/pipenv/project.py b/pipenv/project.py index 4f947e4b..f58b9c58 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -585,12 +585,13 @@ class Project(object): else: empty_inline_table = toml.TomlDecoder().get_empty_inline_table for section in ("packages", "dev-packages"): - table_data = parsed.get(section, {}) + table_data = parsed.get(section, {}).copy() for package, value in table_data.items(): if hasattr(value, "keys"): table = empty_inline_table() table.update(value) table_data[package] = table + parsed[section] = table_data return parsed def _parse_pipfile(self, contents): diff --git a/pipenv/utils.py b/pipenv/utils.py index c9feeafd..8674aee2 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -468,6 +468,73 @@ def create_spinner(text, nospin=None, spinner_name=None): yield sp +def resolve(cmd, sp): + from .vendor import delegator + from .cmdparse import Script + from .vendor.pexpect.exceptions import EOF, TIMEOUT + from .vendor.vistir.compat import to_native_string + EOF.__module__ = "pexpect.exceptions" + from ._compat import decode_output + c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) + _out = decode_output("") + result = None + out = to_native_string("") + while True: + try: + result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT) + except (EOF, TIMEOUT): + pass + if result is None: + break + _out = c.subprocess.before + if _out is not None: + _out = decode_output("{0}".format(_out)) + out += _out + sp.text = to_native_string("{0}".format(_out[:100])) + if environments.is_verbose(): + if _out is not None: + sp._hide_cursor() + sp.write(_out.rstrip()) + sp._show_cursor() + c.block() + if c.return_code != 0: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Locking Failed!" + )) + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) + sys.exit(c.return_code) + return c + + +def get_locked_dep(dep, pipfile_section): + entry = None + cleaner_kwargs = { + "is_top_level": False, + "pipfile_entry": None + } + if isinstance(dep, Mapping) and dep.get("name", ""): + name_options = [dep["name"], pep423_name(dep["name"])] + name = next(iter(k for k in name_options if k in pipfile_section), None) + entry = pipfile_section[name] if name else None + + if entry: + cleaner_kwargs.update({"is_top_level": True, "pipfile_entry": entry}) + lockfile_entry = clean_resolved_dep(dep, **cleaner_kwargs) + return lockfile_entry + + +def prepare_lockfile(results, pipfile, lockfile): + from .vendor.requirementslib.utils import is_vcs + for dep in results: + # Merge in any relevant information from the pipfile entry, including + # markers, normalized names, URL info, etc that we may have dropped during lock + if not is_vcs(dep): + lockfile_entry = get_locked_dep(dep, pipfile) + lockfile.update(lockfile_entry) + return lockfile + + def venv_resolve_deps( deps, which, @@ -476,21 +543,42 @@ def venv_resolve_deps( clear=False, allow_global=False, pypi_mirror=None, + dev=False, + pipfile=None, + lockfile=None ): from .vendor.vistir.misc import fs_str from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError from .vendor.vistir.path import create_tracked_tempdir - from .cmdparse import Script - from .vendor.pexpect.exceptions import EOF, TIMEOUT - from .vendor import delegator from . import resolver - from ._compat import decode_output import json - if not deps: - return [] + vcs_deps = [] + vcs_lockfile = {} + results = [] + pipfile_section = "dev_packages" if dev else "packages" + lockfile_section = "develop" if dev else "default" + vcs_section = "vcs_{0}".format(pipfile_section) + vcs_deps = getattr(project, vcs_section, []) + if not deps and not vcs_deps: + return {} + if not pipfile: + pipfile = getattr(project, pipfile_section, None) + if not lockfile: + lockfile = project._lockfile req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") + if vcs_deps: + with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: + vcs_reqs, vcs_lockfile = get_vcs_deps( + project, + which=which, + clear=clear, + pre=pre, + allow_global=allow_global, + dev=dev, + ) + vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() @@ -509,48 +597,39 @@ def venv_resolve_deps( os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") - out = to_native_string("") - EOF.__module__ = "pexpect.exceptions" with create_spinner(text=fs_str("Locking...")) as sp: - c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) - _out = decode_output("") - result = None - while True: - try: - result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT) - except (EOF, TIMEOUT): - pass - if result is None: - break - _out = c.subprocess.before - if _out is not None: - _out = decode_output("{0}".format(_out)) - out += _out - sp.text = to_native_string("{0}".format(_out[:100])) - if environments.is_verbose(): - if _out is not None: - sp._hide_cursor() - sp.write(_out.rstrip()) - sp._show_cursor() - c.block() - if c.return_code != 0: - sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( - "Locking Failed!" - )) - click_echo(c.out.strip(), err=True) - click_echo(c.err.strip(), err=True) - sys.exit(c.return_code) + c = resolve(cmd, sp) + results = c.out + if vcs_deps: + with temp_environ(): + os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) + sp.text = to_native_string("Locking VCS Dependencies...") + vcs_c = resolve(cmd, sp) + vcs_results, vcs_err = vcs_c.out, vcs_c.err else: - sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + vcs_results, vcs_err = "", "" + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + outputs = [results, vcs_results] if environments.is_verbose(): - click_echo(c.out.split("RESULTS:")[0], err=True) + for output in outputs: + click_echo(output.split("RESULTS:")[0], err=True) try: - return json.loads(c.out.split("RESULTS:")[1].strip()) + results = json.loads(results.split("RESULTS:")[1].strip()) + if vcs_results: + # For vcs dependencies, treat the initial pass at locking (i.e. checkout) + # as the pipfile entry because it gets us an actual ref to use + vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) + vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile) + else: + vcs_results = [] except (IndexError, JSONDecodeError): - click_echo(c.out.strip(), err=True) - click_echo(c.err.strip(), err=True) + for out, err in [(c.out, c.err), (vcs_results, vcs_err)]: + click_echo(out.strip(), err=True) + click_echo(err.strip(), err=True) raise RuntimeError("There was a problem with locking.") + lockfile[lockfile_section] = prepare_lockfile(results, pipfile, lockfile[lockfile_section]) + lockfile[lockfile_section].update(vcs_lockfile) def resolve_deps( diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 80e029d9..c3c2d59f 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -1004,6 +1004,9 @@ class InlineTable(Item, dict): if key is not None: super(InlineTable, self).__setitem__(key, value) + if hasattr(value, "trivia") and value.trivia.comment: + value.trivia.comment = "" + m = re.match("(?s)^[^ ]*([ ]+).*$", self._trivia.indent) if not m: return diff --git a/tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch b/tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch index 755bd31a..8cd6d5ca 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-dump-inline-table.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py -index 781e2e98..80e029d9 100644 +index 781e2e98..c3c2d59f 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -21,6 +21,7 @@ if PY2: @@ -22,3 +22,13 @@ index 781e2e98..80e029d9 100644 for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): val[k] = item(v, _parent=val) +@@ -1000,6 +1004,9 @@ class InlineTable(Item, dict): + if key is not None: + super(InlineTable, self).__setitem__(key, value) + ++ if hasattr(value, "trivia") and value.trivia.comment: ++ value.trivia.comment = "" ++ + m = re.match("(?s)^[^ ]*([ ]+).*$", self._trivia.indent) + if not m: + return