Avoid wait() on processes

.azure-pipelines/steps/run-tests-windows.yml

@@ -43,10 +43,11 @@ steps:
     parameters:
       python_version: ${{ parameters.python_version }}
 
-  - script: |
+  - powershell: |
       git submodule sync
       git submodule update --init --recursive
-      pipenv run pytest -ra -n 4 -v --junit-xml=junit/test-results.xml tests/
+      $venv = (pipenv --venv)[0]
+      & $venv/Scripts/pytest.exe -ra -n 4 -v --junit-xml=junit/test-results.xml tests/
     failOnStderr: false
     displayName: Run integration tests
     env:

pipenv/core.py

@@ -7,7 +7,6 @@ import time
 import warnings
 
 import click
-
 import dotenv
 import pipfile
 import vistir
@@ -18,7 +17,7 @@ from pipenv import environments, exceptions, pep508checker, progress
 from pipenv._compat import decode_for_output, fix_utf8
 from pipenv.environments import (
     PIP_EXISTS_ACTION, PIPENV_CACHE_DIR, PIPENV_COLORBLIND,
-    PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_DONT_USE_PYENV, PIPENV_DONT_USE_ASDF,
+    PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_DONT_USE_ASDF, PIPENV_DONT_USE_PYENV,
     PIPENV_HIDE_EMOJIS, PIPENV_MAX_SUBPROCESS, PIPENV_PYUP_API_KEY,
     PIPENV_RESOLVE_VCS, PIPENV_SHELL_FANCY, PIPENV_SKIP_VALIDATION, PIPENV_YES,
     SESSION_IS_INTERACTIVE, is_type_checking
@@ -28,15 +27,16 @@ from pipenv.project import Project
 from pipenv.utils import (
     convert_deps_to_pip, create_spinner, download_file,
     escape_grouped_arguments, find_python, find_windows_executable,
-    get_canonical_names, get_source_list, interrupt_handled_subprocess,
-    is_pinned, is_python_command, is_required_version, is_star, is_valid_url,
-    parse_indexes, pep423_name, prepare_pip_source_args, proper_case,
-    python_version, run_command, subprocess_run, venv_resolve_deps
+    get_canonical_names, get_source_list, is_pinned, is_python_command,
+    is_required_version, is_star, is_valid_url, parse_indexes, pep423_name,
+    prepare_pip_source_args, proper_case, python_version, run_command,
+    subprocess_run, venv_resolve_deps
 )
 
 
 if is_type_checking():
     from typing import Dict, List, Optional, Union
+
     from pipenv.vendor.requirementslib.models.requirements import Requirement
     TSourceDict = Dict[str, Union[str, bool]]
 
@@ -394,7 +394,7 @@ def ensure_python(three=None, python=None):
             err=True,
         )
         # check for python installers
-        from .installers import Pyenv, Asdf, InstallerError, InstallerNotFound
+        from .installers import Asdf, InstallerError, InstallerNotFound, Pyenv
 
         # prefer pyenv if both pyenv and asdf are installed as it's
         # dedicated to python installs so probably the preferred
@@ -713,7 +713,9 @@ def batch_install(deps_list, procs, failed_deps_queue,
                   requirements_dir, no_deps=True, ignore_hashes=False,
                   allow_global=False, blocking=False, pypi_mirror=None,
                   retry=True, sequential_deps=None):
-    from .vendor.requirementslib.models.utils import strip_extras_markers_from_requirement
+    from .vendor.requirementslib.models.utils import (
+        strip_extras_markers_from_requirement
+    )
     if sequential_deps is None:
         sequential_deps = []
     failed = (not retry)
@@ -766,7 +768,8 @@ def batch_install(deps_list, procs, failed_deps_queue,
             if failed and not dep.is_vcs:
                 use_pep517 = getattr(dep, "use_pep517", False)
 
-            is_blocking = any([dep.editable, dep.is_vcs, blocking])
+            is_sequential = sequential_deps and dep.name in sequential_dep_names
+            is_blocking = any([dep.editable, dep.is_vcs, blocking, is_sequential])
             c = pip_install(
                 dep,
                 ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]),
@@ -781,10 +784,6 @@ def batch_install(deps_list, procs, failed_deps_queue,
                 use_pep517=use_pep517,
             )
             c.dep = dep
-            # if dep.is_vcs or dep.editable:
-            is_sequential = sequential_deps and dep.name in sequential_dep_names
-            if is_sequential and not is_blocking:
-                c.wait()
 
             procs.put(c)
             if procs.full() or procs.qsize() == len(deps_list) or is_sequential:
@@ -963,13 +962,13 @@ def do_create_virtualenv(python=None, site_packages=None, pypi_mirror=None):
     # Actually create the virtualenv.
     error = None
     with create_spinner("Creating virtual environment...") as sp:
-        with interrupt_handled_subprocess(cmd, combine_stderr=False, env=pip_config) as c:
-            click.echo(crayons.cyan(f"{c.out}"), err=True)
-            if c.returncode != 0:
-                error = c.err if environments.is_verbose() else exceptions.prettify_exc(c.err)
-                sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Failed creating virtual environment"))
-            else:
-                sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Successfully created virtual environment!"))
+        c = subprocess_run(cmd, env=pip_config)
+        click.echo(crayons.cyan(f"{c.stdout}"), err=True)
+        if c.returncode != 0:
+            error = c.stderr if environments.is_verbose() else exceptions.prettify_exc(c.stderr)
+            sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Failed creating virtual environment"))
+        else:
+            sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Successfully created virtual environment!"))
     if error is not None:
         raise exceptions.VirtualenvCreationException(
             extra=crayons.red(f"{error}")
@@ -1022,7 +1021,7 @@ def get_downloads_info(names_map, section):
         version = parse_download_fname(fname, name)
         # Get the hash of each file.
         cmd = [
-            escape_grouped_arguments(which_pip()),
+            which_pip(),
             "hash",
             os.sep.join([project.download_location, fname]),
         ]
@@ -1178,10 +1177,9 @@ def do_purge(bare=False, downloads=False, allow_global=False):
         )
 
     command = [
-        escape_grouped_arguments(which_pip(allow_global=allow_global)),
+        which_pip(allow_global=allow_global),
         "uninstall", "-y",
-        " ".join(to_remove),
-    ]
+    ] + list(to_remove)
     if environments.is_verbose():
         click.echo(f"$ {' '.join(command)}")
     c = subprocess_run(command)
@@ -1210,7 +1208,8 @@ def do_init(
 ):
     """Executes the init functionality."""
     from .environments import (
-        PIPENV_VIRTUALENV, PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_PYTHON, PIPENV_USE_SYSTEM
+        PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_PYTHON, PIPENV_USE_SYSTEM,
+        PIPENV_VIRTUALENV
     )
     python = None
     if PIPENV_PYTHON is not None:
@@ -1536,7 +1535,7 @@ def pip_download(package_name):
     }
     for source in project.sources:
         cmd = [
-            escape_grouped_arguments(which_pip()),
+            which_pip(),
             "download",
             package_name,
             "-i", source["url"],
@@ -1795,11 +1794,12 @@ def do_py(system=False):
 
 def do_outdated(pypi_mirror=None, pre=False, clear=False):
     # TODO: Allow --skip-lock here?
+    from collections import namedtuple
+
+    from .vendor.packaging.utils import canonicalize_name
     from .vendor.requirementslib.models.requirements import Requirement
     from .vendor.requirementslib.models.utils import get_version
-    from .vendor.packaging.utils import canonicalize_name
     from .vendor.vistir.compat import Mapping
-    from collections import namedtuple
 
     packages = {}
     package_info = namedtuple("PackageInfo", ["name", "installed", "available"])
@@ -1892,7 +1892,7 @@ def do_install(
     selective_upgrade=False,
     site_packages=None,
 ):
-    from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM
+    from .environments import PIPENV_USE_SYSTEM, PIPENV_VIRTUALENV
     from .vendor.pip_shims.shims import PipError
 
     requirements_directory = vistir.path.create_tracked_tempdir(
@@ -2211,8 +2211,8 @@ def do_uninstall(
     ctx=None
 ):
     from .environments import PIPENV_USE_SYSTEM
-    from .vendor.requirementslib.models.requirements import Requirement
     from .vendor.packaging.utils import canonicalize_name
+    from .vendor.requirementslib.models.requirements import Requirement
 
     # Automatically use an activated virtualenv.
     if PIPENV_USE_SYSTEM:
@@ -2559,8 +2559,8 @@ def do_check(
     args=None,
     pypi_mirror=None
 ):
-    from pipenv.vendor.vistir.compat import JSONDecodeError
     from pipenv.vendor.first import first
+    from pipenv.vendor.vistir.compat import JSONDecodeError
 
     if not system:
         # Ensure that virtualenv is available.
@@ -2707,8 +2707,8 @@ def do_check(
 
 
 def do_graph(bare=False, json=False, json_tree=False, reverse=False):
-    from pipenv.vendor.vistir.compat import JSONDecodeError
     from pipenv.vendor import pipdeptree
+    from pipenv.vendor.vistir.compat import JSONDecodeError
     pipdeptree_path = pipdeptree.__file__.rstrip("cdo")
     try:
         python_path = which("python")
@@ -2778,7 +2778,9 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False):
             err=True,
         )
         sys.exit(1)
-    cmd_args = [python_path, pipdeptree_path, flag, "-l"]
+    cmd_args = [python_path, pipdeptree_path, "-l"]
+    if flag:
+        cmd_args.append(flag)
     c = run_command(cmd_args)
     # Run dep-tree.
     if not bare:

pipenv/environment.py

@@ -17,7 +17,7 @@ from pipenv.vendor.cached_property import cached_property
 from pipenv.vendor.packaging.utils import canonicalize_name
 from pipenv.vendor import vistir
 
-from pipenv.utils import normalize_path, make_posix
+from pipenv.utils import normalize_path, make_posix, subprocess_run
 
 
 if False:
@@ -361,9 +361,7 @@ class Environment:
         tmpfile_path = make_posix(tmpfile.name)
         py_command = self.build_command(python_lib=True, python_inc=True, scripts=True, py_version=True)
         command = [self.python, "-c", py_command.format(tmpfile_path)]
-        c = vistir.misc.run(
-            command, return_object=True, block=True, nospin=True, write_to_stdout=False
-        )
+        c = subprocess_run(command)
         if c.returncode == 0:
             paths = {}
             with open(tmpfile_path, "r", encoding="utf-8") as fh:
@@ -375,8 +373,8 @@ class Environment:
                     paths[key] = make_posix(paths[key])
             return paths
         else:
-            vistir.misc.echo(f"Failed to load paths: {c.err}", fg="yellow")
-            vistir.misc.echo(f"Output: {c.out}", fg="yellow")
+            vistir.misc.echo(f"Failed to load paths: {c.stderr}", fg="yellow")
+            vistir.misc.echo(f"Output: {c.stdout}", fg="yellow")
         return None
 
     def get_lib_paths(self):
@@ -391,9 +389,7 @@ class Environment:
         tmpfile_path = make_posix(tmpfile.name)
         py_command = self.build_command(python_lib=True)
         command = [self.python, "-c", py_command.format(tmpfile_path)]
-        c = vistir.misc.run(
-            command, return_object=True, block=True, nospin=True, write_to_stdout=False
-        )
+        c = subprocess_run(command)
         paths = None
         if c.returncode == 0:
             paths = {}
@@ -406,8 +402,8 @@ class Environment:
                     paths[key] = make_posix(paths[key])
             return paths
         else:
-            vistir.misc.echo(f"Failed to load paths: {c.err}", fg="yellow")
-            vistir.misc.echo(f"Output: {c.out}", fg="yellow")
+            vistir.misc.echo(f"Failed to load paths: {c.stderr}", fg="yellow")
+            vistir.misc.echo(f"Output: {c.stdout}", fg="yellow")
         if not paths:
             if not self.prefix.joinpath("lib").exists():
                 return {}
@@ -445,9 +441,7 @@ class Environment:
             "fh = io.open('{0}', 'w'); fh.write(value); fh.close()"
         )
         command = [self.python, "-c", py_command.format(tmpfile_path)]
-        c = vistir.misc.run(
-            command, return_object=True, block=True, nospin=True, write_to_stdout=False
-        )
+        c = subprocess_run(command)
         if c.returncode == 0:
             paths = []
             with open(tmpfile_path, "r", encoding="utf-8") as fh:
@@ -457,8 +451,8 @@ class Environment:
                     paths[key] = make_posix(paths[key])
             return paths
         else:
-            vistir.misc.echo(f"Failed to load paths: {c.err}", fg="yellow")
-            vistir.misc.echo(f"Output: {c.out}", fg="yellow")
+            vistir.misc.echo(f"Failed to load paths: {c.stderr}", fg="yellow")
+            vistir.misc.echo(f"Output: {c.stdout}", fg="yellow")
         return None
 
     @cached_property
@@ -472,8 +466,8 @@ class Environment:
         """
 
         command = [self.python, "-c", "import sys; print(sys.prefix)"]
-        c = vistir.misc.run(command, return_object=True, block=True, nospin=True, write_to_stdout=False)
-        sys_prefix = vistir.compat.Path(vistir.misc.to_text(c.out).strip()).as_posix()
+        c = subprocess_run(command)
+        sys_prefix = vistir.compat.Path(c.stdout.strip()).as_posix()
         return sys_prefix
 
     @cached_property

setup.cfg

@@ -32,11 +32,9 @@ lines_after_imports=2
 lines_between_types=1
 multi_line_output=5
 line_length=80
-not_skip=__init__.py
 known_first_party =
     pipenv
     tests
-ignore_trailing_comma=true
 
 [mypy]
 ignore_missing_imports=true

tests/integration/test_cli.py

@@ -85,14 +85,14 @@ def test_pipenv_graph(PipenvInstance):
         c = p.pipenv('install tablib')
         assert c.returncode == 0
         graph = p.pipenv("graph")
-        assert graph.ok
-        assert "tablib" in graph.out
+        assert graph.returncode == 0
+        assert "tablib" in graph.stdout
         graph_json = p.pipenv("graph --json")
-        assert graph_json.ok
-        assert "tablib" in graph_json.out
+        assert graph_json.returncode == 0
+        assert "tablib" in graph_json.stdout
         graph_json_tree = p.pipenv("graph --json-tree")
-        assert graph_json_tree.ok
-        assert "tablib" in graph_json_tree.out
+        assert graph_json_tree.returncode == 0
+        assert "tablib" in graph_json_tree.stdout
 
 
 @pytest.mark.cli
@@ -186,13 +186,13 @@ def test_pipenv_clean_pip_warnings(PipenvInstance):
 @pytest.mark.cli
 def test_venv_envs(PipenvInstance):
     with PipenvInstance() as p:
-        assert p.pipenv('--envs').out
+        assert p.pipenv('--envs').stdout
 
 
 @pytest.mark.cli
 def test_bare_output(PipenvInstance):
     with PipenvInstance() as p:
-        assert p.pipenv('').out
+        assert p.pipenv('').stdout
 
 
 @pytest.mark.cli
@@ -212,7 +212,7 @@ pyver = "which python"
 @pytest.mark.cli
 def test_help(PipenvInstance):
     with PipenvInstance() as p:
-        assert p.pipenv('--help').out
+        assert p.pipenv('--help').stdout
 
 
 @pytest.mark.cli

tests/integration/test_dot_venv.py

@@ -15,7 +15,7 @@ def test_venv_in_project(PipenvInstance):
         with PipenvInstance() as p:
             c = p.pipenv('install requests')
             assert c.returncode == 0
-            assert normalize_drive(p.path) in p.pipenv('--venv').out
+            assert normalize_drive(p.path) in p.pipenv('--venv').stdout
 
 
 @pytest.mark.dotvenv
@@ -25,12 +25,12 @@ def test_venv_at_project_root(PipenvInstance):
             os.environ['PIPENV_VENV_IN_PROJECT'] = '1'
             c = p.pipenv('install')
             assert c.returncode == 0
-            assert normalize_drive(p.path) in p.pipenv('--venv').out
+            assert normalize_drive(p.path) in p.pipenv('--venv').stdout
             del os.environ['PIPENV_VENV_IN_PROJECT']
             os.mkdir('subdir')
             os.chdir('subdir')
             # should still detect installed
-            assert normalize_drive(p.path) in p.pipenv('--venv').out
+            assert normalize_drive(p.path) in p.pipenv('--venv').stdout
 
 
 @pytest.mark.dotvenv
@@ -39,7 +39,7 @@ def test_reuse_previous_venv(PipenvInstance):
         os.mkdir('.venv')
         c = p.pipenv('install requests')
         assert c.returncode == 0
-        assert normalize_drive(p.path) in p.pipenv('--venv').out
+        assert normalize_drive(p.path) in p.pipenv('--venv').stdout
 
 
 @pytest.mark.dotvenv

tests/integration/test_lock.py

@@ -48,13 +48,13 @@ flask = "==0.12.2"
         c = p.pipenv('lock -r')
         d = p.pipenv('lock -r -d')
         assert c.returncode == 0
-        assert d.return_code == 0
+        assert d.returncode == 0
 
         for req in req_list:
             assert req in c.stdout
 
         for req in dev_req_list:
-            assert req in d.out
+            assert req in d.stdout
 
 
 @pytest.mark.lock