Merge branch 'bugfix/3148' into feature/keep-outdated-peep

Signed-off-by: Dan Ryan <dan@danryan.co>

.azure-pipelines/jobs/run-tests.yml

@@ -16,8 +16,9 @@ steps:
     export PIP_PROCESS_DEPENDENCY_LINKS="1"
     echo "Path $PATH"
     echo "Installing Pipenv…"
-    pip install -e "$(pwd)" --upgrade
+    pip install -e "$(pwd)[test]" --upgrade
     pipenv install --deploy --dev
+    pipenv run pip install -e "$(pwd)[test]" --upgrade
     echo pipenv --venv && echo pipenv --py && echo pipenv run python --version
   displayName: Make Virtualenv
 

.azure-pipelines/steps/create-virtualenv.yml

@@ -1,6 +1,6 @@
 steps:
 - script: |
     virtualenv D:\.venv
-    D:\.venv\Scripts\pip.exe install -e . && D:\.venv\Scripts\pipenv install --dev
+    D:\.venv\Scripts\pip.exe install -e .[test] && D:\.venv\Scripts\pipenv install --dev && D:\.venv\Scripts\pipenv run pip install -e .[test]
     echo D:\.venv\Scripts\pipenv --venv && echo D:\.venv\Scripts\pipenv --py && echo D:\.venv\Scripts\pipenv run python --version
   displayName: Make Virtualenv

pipenv/cli/command.py

@@ -300,6 +300,7 @@ def uninstall(
     if retcode:
         sys.exit(retcode)
 
+
 @cli.command(short_help="Generates Pipfile.lock.", context_settings=CONTEXT_SETTINGS)
 @lock_options
 @pass_state
@@ -399,8 +400,8 @@ def shell(
 @pass_state
 def run(state, command, args):
     """Spawns a command installed into the virtualenv."""
-    from ..core import do_run
-
+    from ..core import do_run, warn_in_virtualenv
+    warn_in_virtualenv()
     do_run(
         command=command, args=args, three=state.three, python=state.python, pypi_mirror=state.pypi_mirror
     )
@@ -629,7 +630,8 @@ def sync(
 def clean(ctx, state, dry_run=False, bare=False, user=False):
     """Uninstalls all packages not specified in Pipfile.lock."""
     from ..core import do_clean
-    do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run)
+    do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run,
+             system=state.system)
 
 
 # Only invoke the "did you mean" when an argument wasn't passed (it breaks those).

pipenv/core.py

@@ -2721,34 +2721,32 @@ def do_sync(
         click.echo(crayons.green("All dependencies are now up-to-date!"))
 
 
-def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None):
+def do_clean(
+    ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None,
+    system=False
+):
     # Ensure that virtualenv is available.
     from packaging.utils import canonicalize_name
     ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror)
     ensure_lockfile(pypi_mirror=pypi_mirror)
     # Make sure that the virtualenv's site packages are configured correctly
     # otherwise we may end up removing from the global site packages directory
-    installed_package_names = [
-        canonicalize_name(pkg.project_name) for pkg
-        in project.environment.get_installed_packages()
-    ]
+    installed_package_names = project.installed_package_names.copy()
     # Remove known "bad packages" from the list.
     for bad_package in BAD_PACKAGES:
         if canonicalize_name(bad_package) in installed_package_names:
             if environments.is_verbose():
                 click.echo("Ignoring {0}.".format(bad_package), err=True)
-            del installed_package_names[installed_package_names.index(
-                canonicalize_name(bad_package)
-            )]
+            installed_package_names.remove(canonicalize_name(bad_package))
     # Intelligently detect if --dev should be used or not.
-    develop = [canonicalize_name(k) for k in project.lockfile_content["develop"].keys()]
-    default = [canonicalize_name(k) for k in project.lockfile_content["default"].keys()]
-    for used_package in set(develop + default):
+    locked_packages = {
+        canonicalize_name(pkg) for pkg in project.lockfile_package_names["combined"]
+    }
+    for used_package in locked_packages:
         if used_package in installed_package_names:
-            del installed_package_names[installed_package_names.index(
-                canonicalize_name(used_package)
-            )]
+            installed_package_names.remove(used_package)
     failure = False
+    cmd = [which_pip(allow_global=system), "uninstall", "-y", "-qq"]
     for apparent_bad_package in installed_package_names:
         if dry_run and not bare:
             click.echo(apparent_bad_package)
@@ -2760,9 +2758,8 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro
                     )
                 )
             # Uninstall the package.
-            c = delegator.run(
-                "{0} uninstall {1} -y".format(which_pip(), apparent_bad_package)
-            )
+            cmd_str = Script.parse(cmd + [apparent_bad_package]).cmdify()
+            c = delegator.run(cmd_str, block=True)
             if c.return_code != 0:
                 failure = True
     sys.exit(int(failure))

pipenv/environment.py

@@ -92,12 +92,16 @@ class Environment(object):
             deps |= cls.resolve_dist(dist, working_set)
         return deps
 
-    def add_dist(self, dist_name):
-        dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name))
+    def extend_dists(self, dist):
         extras = self.resolve_dist(dist, self.base_working_set)
+        self.extra_dists.append(dist)
         if extras:
             self.extra_dists.extend(extras)
 
+    def add_dist(self, dist_name):
+        dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name))
+        self.extend_dists(dist)
+
     @cached_property
     def python_version(self):
         with self.activated():
@@ -244,7 +248,7 @@ class Environment(object):
         """
 
         pkg_resources = self.safe_import("pkg_resources")
-        return pkg_resources.find_distributions(self.paths["PYTHONPATH"])
+        return pkg_resources.find_distributions(self.paths["libdirs"])
 
     def find_egg(self, egg_dist):
         """Find an egg by name in the given environment"""
@@ -271,16 +275,22 @@ class Environment(object):
     def dist_is_in_project(self, dist):
         """Determine whether the supplied distribution is in the environment."""
         from .project import _normalized
-        prefix = _normalized(self.base_paths["prefix"])
+        prefixes = [
+            _normalized(prefix) for prefix in self.base_paths["libdirs"]
+            if _normalized(self.prefix).startswith(_normalized(prefix))
+        ]
         location = self.locate_dist(dist)
         if not location:
             return False
-        return _normalized(location).startswith(prefix)
+        return any(_normalized(location).startswith(prefix) for prefix in prefixes)
 
     def get_installed_packages(self):
         """Returns all of the installed packages in a given environment"""
         workingset = self.get_working_set()
-        packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)]
+        packages = [
+            pkg for pkg in workingset
+            if self.dist_is_in_project(pkg) and pkg.key != "python"
+        ]
         return packages
 
     @contextlib.contextmanager

pipenv/exceptions.py

@@ -235,11 +235,11 @@ class UninstallError(PipenvException):
             crayons.yellow("$ {0!r}".format(command), bold=True)
         )),]
         extra.extend([crayons.blue(line.strip()) for line in return_values.splitlines()])
-        if isinstance(package, (tuple, set, list)):
+        if isinstance(package, (tuple, list, set)):
             package = " ".join(package)
         message = "{0!s} {1!s}...".format(
             crayons.normal("Failed to uninstall package(s)"),
-            crayons.yellow(package, bold=True)
+            crayons.yellow(str(package), bold=True)
         )
         self.exit_code = return_code
         PipenvException.__init__(self, message=fix_utf8(message), extra=extra)

pipenv/project.py

@@ -34,7 +34,7 @@ from .utils import (
     get_canonical_names, get_url_name, get_workon_home, is_editable,
     is_installable_file, is_star, is_valid_url, is_virtual_environment,
     looks_like_dir, normalize_drive, pep423_name, proper_case, python_version,
-    safe_expandvars
+    safe_expandvars, get_pipenv_dist
 )
 
 
@@ -340,7 +340,11 @@ class Project(object):
                 prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile,
                 project=self
             )
-            self._environment.add_dist("pipenv")
+            pipenv_dist = get_pipenv_dist()
+            if pipenv_dist:
+                self._environment.extend_dists(pipenv_dist)
+            else:
+                self._environment.add_dist("pipenv")
         return self._environment
 
     def get_outdated_packages(self):

pipenv/resolver.py

@@ -7,12 +7,48 @@ import sys
 os.environ["PIP_PYTHON_PATH"] = str(sys.executable)
 
 
-def _patch_path():
+def find_site_path(pkg, site_dir=None):
+    import pkg_resources
+    if site_dir is not None:
+        site_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    working_set = pkg_resources.WorkingSet([site_dir] + sys.path[:])
+    for dist in working_set:
+        root = dist.location
+        base_name = dist.project_name if dist.project_name else dist.key
+        name = None
+        if "top_level.txt" in dist.metadata_listdir(""):
+            name = next(iter([l.strip() for l in dist.get_metadata_lines("top_level.txt") if l is not None]), None)
+        if name is None:
+            name = pkg_resources.safe_name(base_name).replace("-", "_")
+        if not any(pkg == _ for _ in [base_name, name]):
+            continue
+        path_options = [name, "{0}.py".format(name)]
+        path_options = [os.path.join(root, p) for p in path_options if p is not None]
+        path = next(iter(p for p in path_options if os.path.exists(p)), None)
+        if path is not None:
+            return (dist, path)
+    return (None, None)
+
+
+def _patch_path(pipenv_site=None):
     import site
     pipenv_libdir = os.path.dirname(os.path.abspath(__file__))
     pipenv_site_dir = os.path.dirname(pipenv_libdir)
-    site.addsitedir(pipenv_site_dir)
-    for _dir in ("vendor", "patched"):
+    pipenv_dist = None
+    if pipenv_site is not None:
+        pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site)
+    else:
+        pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site_dir)
+    if pipenv_dist is not None:
+        pipenv_dist.activate()
+    else:
+        site.addsitedir(next(iter(
+            sitedir for sitedir in (pipenv_site, pipenv_site_dir)
+            if sitedir is not None
+        ), None))
+    if pipenv_path is not None:
+        pipenv_libdir = pipenv_path
+    for _dir in ("vendor", "patched", pipenv_libdir):
         sys.path.insert(0, os.path.join(pipenv_libdir, _dir))
 
 
@@ -25,8 +61,11 @@ def get_parser():
     parser.add_argument("--dev", action="store_true", default=False)
     parser.add_argument("--debug", action="store_true", default=False)
     parser.add_argument("--system", action="store_true", default=False)
+    parser.add_argument("--parse-only", action="store_true", default=False)
+    parser.add_argument("--pipenv-site", metavar="pipenv_site_dir", action="store",
+                        default=os.environ.get("PIPENV_SITE_DIR"))
     parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store",
-                            default=os.environ.get("PIPENV_REQ_DIR"))
+                        default=os.environ.get("PIPENV_REQ_DIR"))
     parser.add_argument("packages", nargs="*")
     return parser
 
@@ -407,12 +446,39 @@ def clean_outdated(results, resolver, project, dev=False):
     return new_results
 
 
-def _main(pre, clear, verbose, system, requirements_dir, dev, packages):
-    os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]])
-    os.environ["PIP_PYTHON_PATH"] = str(sys.executable)
+def parse_packages(packages, pre, clear, system, requirements_dir=None):
+    from pipenv.vendor.requirementslib.models.requirements import Requirement
+    from pipenv.vendor.vistir.contextmanagers import cd, temp_path
+    from pipenv.utils import parse_indexes
+    parsed_packages = []
+    for package in packages:
+        indexes, trusted_hosts, line = parse_indexes(package)
+        line = " ".join(line)
+        pf = dict()
+        req = Requirement.from_line(line)
+        if not req.name:
+            with temp_path(), cd(req.req.setup_info.base_dir):
+                sys.path.insert(0, req.req.setup_info.base_dir)
+                req.req._setup_info.get_info()
+                req.update_name_from_path(req.req.setup_info.base_dir)
+        print(os.listdir(req.req.setup_info.base_dir))
+        try:
+            name, entry = req.pipfile_entry
+        except Exception:
+            continue
+        else:
+            if name is not None and entry is not None:
+                pf[name] = entry
+                parsed_packages.append(pf)
+    print("RESULTS:")
+    if parsed_packages:
+        print(json.dumps(parsed_packages))
+    else:
+        print(json.dumps([]))
 
+
+def resolve_packages(pre, clear, verbose, system, requirements_dir, packages):
     from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources
-
     pypi_mirror_source = (
         create_mirror_source(os.environ["PIPENV_PYPI_MIRROR"])
         if "PIPENV_PYPI_MIRROR" in os.environ
@@ -456,36 +522,46 @@ def _main(pre, clear, verbose, system, requirements_dir, dev, packages):
         print(json.dumps([]))
 
 
-def main():
-    _patch_path()
-    import warnings
-    from pipenv.vendor.vistir.compat import ResourceWarning
-    warnings.simplefilter("ignore", category=ResourceWarning)
-    import io
-    import six
-    if six.PY3:
-        import atexit
-        stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')
-        atexit.register(stdout_wrapper.close)
-        stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8')
-        atexit.register(stderr_wrapper.close)
-        sys.stdout = stdout_wrapper
-        sys.stderr = stderr_wrapper
+def _main(pre, clear, verbose, system, requirements_dir, packages, parse_only=False):
+    os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]])
+    os.environ["PIP_PYTHON_PATH"] = str(sys.executable)
+    if parse_only:
+        parse_packages(
+            packages,
+            pre=pre,
+            clear=clear,
+            system=system,
+            requirements_dir=requirements_dir,
+        )
     else:
-        from pipenv._compat import force_encoding
-        force_encoding()
-    os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1")
-    os.environ["PYTHONIOENCODING"] = str("utf-8")
+        resolve_packages(pre, clear, verbose, system, requirements_dir, packages)
+
+
+def main():
     parser = get_parser()
     parsed, remaining = parser.parse_known_args()
-    # sys.argv = remaining
+    _patch_path(pipenv_site=parsed.pipenv_site)
+    import warnings
+    from pipenv.vendor.vistir.compat import ResourceWarning
+    from pipenv.vendor.vistir.misc import get_wrapped_stream
+    warnings.simplefilter("ignore", category=ResourceWarning)
+    import six
+    if six.PY3:
+        stdout = sys.stdout.buffer
+        stderr = sys.stderr.buffer
+    else:
+        stdout = sys.stdout
+        stderr = sys.stderr
+    sys.stderr = get_wrapped_stream(stderr)
+    sys.stdout = get_wrapped_stream(stdout)
+    from pipenv.vendor import colorama
+    colorama.init()
+    os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1")
+    os.environ["PYTHONIOENCODING"] = str("utf-8")
     parsed = handle_parsed_args(parsed)
     _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system,
-          parsed.requirements_dir, parsed.dev, parsed.packages)
+          parsed.requirements_dir, parsed.packages, parse_only=parsed.parse_only)
 
 
 if __name__ == "__main__":
-    _patch_path()
-    from pipenv.vendor import colorama
-    colorama.init()
     main()

pipenv/utils.py

@@ -552,7 +552,7 @@ class Resolver(object):
         return cleaned_checksums
 
     def collect_hashes(self, ireq):
-        from requests import ConnectionError
+        from .vendor.requests import ConnectionError
         collected_hashes = []
         if ireq in self.hashes:
             collected_hashes += list(self.hashes.get(ireq, []))
@@ -1313,7 +1313,7 @@ def get_canonical_names(packages):
     if not isinstance(packages, Sequence):
         if not isinstance(packages, six.string_types):
             return packages
-        packages = [packages,]
+        packages = [packages]
     return set([canonicalize_name(pkg) for pkg in packages if pkg])
 
 
@@ -1742,11 +1742,11 @@ def parse_indexes(line):
     )
     parser.add_argument(
         "--extra-index-url", "--extra-index",
-        metavar="extra_indexes",action="append",
+        metavar="extra_indexes", action="append",
     )
     parser.add_argument("--trusted-host", metavar="trusted_hosts", action="append")
     args, remainder = parser.parse_known_args(line.split())
-    index = [] if not args.index else [args.index,]
+    index = [] if not args.index else [args.index]
     extra_indexes = [] if not args.extra_index_url else args.extra_index_url
     indexes = index + extra_indexes
     trusted_hosts = args.trusted_host if args.trusted_host else []
@@ -1792,3 +1792,12 @@ def is_url_equal(url, other_url):
     unparsed = parsed_url._replace(auth=None, query=None, fragment=None).url
     unparsed_other = parsed_other_url._replace(auth=None, query=None, fragment=None).url
     return unparsed == unparsed_other
+
+
+def get_pipenv_dist(pkg="pipenv", pipenv_site=None):
+    from .resolver import find_site_path
+    pipenv_libdir = os.path.dirname(os.path.abspath(__file__))
+    if pipenv_site is None:
+        pipenv_site = os.path.dirname(pipenv_libdir)
+    pipenv_dist, _ = find_site_path(pkg, site_dir=pipenv_site)
+    return pipenv_dist

pipenv/vendor/requirementslib/models/setup_info.py

@@ -718,7 +718,7 @@ build-backend = "{1}"
             get_metadata(d, pkg_name=self.name, metadata_type=metadata_type)
             for d in metadata_dirs if os.path.exists(d)
         ]
-        metadata = next(iter(d for d in metadata if d), None)
+        metadata = next(iter(d for d in metadata if d is not None), None)
         if metadata is not None:
             self.populate_metadata(metadata)
 

run-tests.bat

@@ -1,7 +1,7 @@
 rem imdisk  -a -s 964515b -m R: -p "/FS:NTFS /Y"
 
 virtualenv R:\.venv
-R:\.venv\Scripts\pip install -e . --upgrade --upgrade-strategy=only-if-needed
+R:\.venv\Scripts\pip install -e .[test] --upgrade --upgrade-strategy=only-if-needed
 R:\.venv\Scripts\pipenv install --dev
 git submodule sync && git submodule update --init --recursive
 SET RAM_DISK=R: && R:\.venv\Scripts\pipenv run pytest -n auto -v tests --tap-stream > report.tap

run-tests.sh

@@ -53,8 +53,8 @@ fi
 echo "Installing dependencies…"
 PIPENV_PYTHON=2.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --dev
 PIPENV_PYTHON=3.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --dev
-PIPENV_PYTHON=2.7 python3 -m pipenv run pip install --upgrade -e .
-PIPENV_PYTHON=3.7 python3 -m pipenv run pip install --upgrade -e .
+PIPENV_PYTHON=2.7 python3 -m pipenv run pip install --upgrade -e .[test]
+PIPENV_PYTHON=3.7 python3 -m pipenv run pip install --upgrade -e .[test]
 echo "$ git submodule sync && git submodule update --init --recursive"
 
 git submodule sync && git submodule update --init --recursive

tests/integration/test_pipenv.py

@@ -101,9 +101,13 @@ def test_directory_with_leading_dash(PipenvInstance):
 
     with mock.patch('pipenv.vendor.vistir.compat.mkdtemp', side_effect=mocked_mkdtemp):
         with temp_environ(), PipenvInstance(chdir=True) as p:
-            del os.environ['PIPENV_VENV_IN_PROJECT']
-            p.pipenv('--python python')
-            venv_path = p.pipenv('--venv').out.strip()
+            if "PIPENV_VENV_IN_PROJECT" in os.environ:
+                del os.environ['PIPENV_VENV_IN_PROJECT']
+            c = p.pipenv('--python python')
+            assert c.return_code == 0
+            c = p.pipenv('--venv')
+            assert c.return_code == 0
+            venv_path = c.out.strip()
             assert os.path.isdir(venv_path)
             # Manually clean up environment, since PipenvInstance assumes that
             # the virutalenv is in the project directory.

tests/integration/test_project.py

@@ -183,7 +183,7 @@ def test_run_in_virtualenv_with_global_context(PipenvInstance, pypi, virtualenv)
         assert c.return_code == 0
         c = p.pipenv('run python -c "import click;print(click.__file__)"')
         assert c.return_code == 0
-        assert c.out.strip().startswith(str(virtualenv))
+        assert c.out.strip().startswith(virtualenv.as_posix())
         c = p.pipenv("clean --dry-run")
         assert c.return_code == 0
         assert "click" in c.out