diff --git a/.azure-pipelines/jobs/run-tests.yml b/.azure-pipelines/jobs/run-tests.yml index 3544af41..2602cba6 100644 --- a/.azure-pipelines/jobs/run-tests.yml +++ b/.azure-pipelines/jobs/run-tests.yml @@ -16,8 +16,9 @@ steps: export PIP_PROCESS_DEPENDENCY_LINKS="1" echo "Path $PATH" echo "Installing Pipenv…" - pip install -e "$(pwd)" --upgrade + pip install -e "$(pwd)[test]" --upgrade pipenv install --deploy --dev + pipenv run pip install -e "$(pwd)[test]" --upgrade echo pipenv --venv && echo pipenv --py && echo pipenv run python --version displayName: Make Virtualenv diff --git a/.azure-pipelines/steps/create-virtualenv.yml b/.azure-pipelines/steps/create-virtualenv.yml index 9d1a6903..60ade40b 100644 --- a/.azure-pipelines/steps/create-virtualenv.yml +++ b/.azure-pipelines/steps/create-virtualenv.yml @@ -1,6 +1,6 @@ steps: - script: | virtualenv D:\.venv - D:\.venv\Scripts\pip.exe install -e . && D:\.venv\Scripts\pipenv install --dev + D:\.venv\Scripts\pip.exe install -e .[test] && D:\.venv\Scripts\pipenv install --dev && D:\.venv\Scripts\pipenv run pip install -e .[test] echo D:\.venv\Scripts\pipenv --venv && echo D:\.venv\Scripts\pipenv --py && echo D:\.venv\Scripts\pipenv run python --version displayName: Make Virtualenv diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index ec1bef61..16ce115e 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -300,6 +300,7 @@ def uninstall( if retcode: sys.exit(retcode) + @cli.command(short_help="Generates Pipfile.lock.", context_settings=CONTEXT_SETTINGS) @lock_options @pass_state @@ -399,8 +400,8 @@ def shell( @pass_state def run(state, command, args): """Spawns a command installed into the virtualenv.""" - from ..core import do_run - + from ..core import do_run, warn_in_virtualenv + warn_in_virtualenv() do_run( command=command, args=args, three=state.three, python=state.python, pypi_mirror=state.pypi_mirror ) @@ -629,7 +630,8 @@ def sync( def clean(ctx, state, dry_run=False, bare=False, user=False): """Uninstalls all packages not specified in Pipfile.lock.""" from ..core import do_clean - do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run) + do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run, + system=state.system) # Only invoke the "did you mean" when an argument wasn't passed (it breaks those). diff --git a/pipenv/core.py b/pipenv/core.py index 08179fd9..6eaca769 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2721,34 +2721,32 @@ def do_sync( click.echo(crayons.green("All dependencies are now up-to-date!")) -def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None): +def do_clean( + ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None, + system=False +): # Ensure that virtualenv is available. from packaging.utils import canonicalize_name ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) ensure_lockfile(pypi_mirror=pypi_mirror) # Make sure that the virtualenv's site packages are configured correctly # otherwise we may end up removing from the global site packages directory - installed_package_names = [ - canonicalize_name(pkg.project_name) for pkg - in project.environment.get_installed_packages() - ] + installed_package_names = project.installed_package_names.copy() # Remove known "bad packages" from the list. for bad_package in BAD_PACKAGES: if canonicalize_name(bad_package) in installed_package_names: if environments.is_verbose(): click.echo("Ignoring {0}.".format(bad_package), err=True) - del installed_package_names[installed_package_names.index( - canonicalize_name(bad_package) - )] + installed_package_names.remove(canonicalize_name(bad_package)) # Intelligently detect if --dev should be used or not. - develop = [canonicalize_name(k) for k in project.lockfile_content["develop"].keys()] - default = [canonicalize_name(k) for k in project.lockfile_content["default"].keys()] - for used_package in set(develop + default): + locked_packages = { + canonicalize_name(pkg) for pkg in project.lockfile_package_names["combined"] + } + for used_package in locked_packages: if used_package in installed_package_names: - del installed_package_names[installed_package_names.index( - canonicalize_name(used_package) - )] + installed_package_names.remove(used_package) failure = False + cmd = [which_pip(allow_global=system), "uninstall", "-y", "-qq"] for apparent_bad_package in installed_package_names: if dry_run and not bare: click.echo(apparent_bad_package) @@ -2760,9 +2758,8 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro ) ) # Uninstall the package. - c = delegator.run( - "{0} uninstall {1} -y".format(which_pip(), apparent_bad_package) - ) + cmd_str = Script.parse(cmd + [apparent_bad_package]).cmdify() + c = delegator.run(cmd_str, block=True) if c.return_code != 0: failure = True sys.exit(int(failure)) diff --git a/pipenv/environment.py b/pipenv/environment.py index 8521501f..fceecb2a 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -92,12 +92,16 @@ class Environment(object): deps |= cls.resolve_dist(dist, working_set) return deps - def add_dist(self, dist_name): - dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + def extend_dists(self, dist): extras = self.resolve_dist(dist, self.base_working_set) + self.extra_dists.append(dist) if extras: self.extra_dists.extend(extras) + def add_dist(self, dist_name): + dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + self.extend_dists(dist) + @cached_property def python_version(self): with self.activated(): @@ -244,7 +248,7 @@ class Environment(object): """ pkg_resources = self.safe_import("pkg_resources") - return pkg_resources.find_distributions(self.paths["PYTHONPATH"]) + return pkg_resources.find_distributions(self.paths["libdirs"]) def find_egg(self, egg_dist): """Find an egg by name in the given environment""" @@ -271,16 +275,22 @@ class Environment(object): def dist_is_in_project(self, dist): """Determine whether the supplied distribution is in the environment.""" from .project import _normalized - prefix = _normalized(self.base_paths["prefix"]) + prefixes = [ + _normalized(prefix) for prefix in self.base_paths["libdirs"] + if _normalized(self.prefix).startswith(_normalized(prefix)) + ] location = self.locate_dist(dist) if not location: return False - return _normalized(location).startswith(prefix) + return any(_normalized(location).startswith(prefix) for prefix in prefixes) def get_installed_packages(self): """Returns all of the installed packages in a given environment""" workingset = self.get_working_set() - packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] + packages = [ + pkg for pkg in workingset + if self.dist_is_in_project(pkg) and pkg.key != "python" + ] return packages @contextlib.contextmanager diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index a055f8a5..e83ab984 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -235,11 +235,11 @@ class UninstallError(PipenvException): crayons.yellow("$ {0!r}".format(command), bold=True) )),] extra.extend([crayons.blue(line.strip()) for line in return_values.splitlines()]) - if isinstance(package, (tuple, set, list)): + if isinstance(package, (tuple, list, set)): package = " ".join(package) message = "{0!s} {1!s}...".format( crayons.normal("Failed to uninstall package(s)"), - crayons.yellow(package, bold=True) + crayons.yellow(str(package), bold=True) ) self.exit_code = return_code PipenvException.__init__(self, message=fix_utf8(message), extra=extra) diff --git a/pipenv/project.py b/pipenv/project.py index 68ec4666..f0119085 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -34,7 +34,7 @@ from .utils import ( get_canonical_names, get_url_name, get_workon_home, is_editable, is_installable_file, is_star, is_valid_url, is_virtual_environment, looks_like_dir, normalize_drive, pep423_name, proper_case, python_version, - safe_expandvars + safe_expandvars, get_pipenv_dist ) @@ -340,7 +340,11 @@ class Project(object): prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile, project=self ) - self._environment.add_dist("pipenv") + pipenv_dist = get_pipenv_dist() + if pipenv_dist: + self._environment.extend_dists(pipenv_dist) + else: + self._environment.add_dist("pipenv") return self._environment def get_outdated_packages(self): diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 468b6027..29672726 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -7,12 +7,48 @@ import sys os.environ["PIP_PYTHON_PATH"] = str(sys.executable) -def _patch_path(): +def find_site_path(pkg, site_dir=None): + import pkg_resources + if site_dir is not None: + site_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + working_set = pkg_resources.WorkingSet([site_dir] + sys.path[:]) + for dist in working_set: + root = dist.location + base_name = dist.project_name if dist.project_name else dist.key + name = None + if "top_level.txt" in dist.metadata_listdir(""): + name = next(iter([l.strip() for l in dist.get_metadata_lines("top_level.txt") if l is not None]), None) + if name is None: + name = pkg_resources.safe_name(base_name).replace("-", "_") + if not any(pkg == _ for _ in [base_name, name]): + continue + path_options = [name, "{0}.py".format(name)] + path_options = [os.path.join(root, p) for p in path_options if p is not None] + path = next(iter(p for p in path_options if os.path.exists(p)), None) + if path is not None: + return (dist, path) + return (None, None) + + +def _patch_path(pipenv_site=None): import site pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) pipenv_site_dir = os.path.dirname(pipenv_libdir) - site.addsitedir(pipenv_site_dir) - for _dir in ("vendor", "patched"): + pipenv_dist = None + if pipenv_site is not None: + pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site) + else: + pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site_dir) + if pipenv_dist is not None: + pipenv_dist.activate() + else: + site.addsitedir(next(iter( + sitedir for sitedir in (pipenv_site, pipenv_site_dir) + if sitedir is not None + ), None)) + if pipenv_path is not None: + pipenv_libdir = pipenv_path + for _dir in ("vendor", "patched", pipenv_libdir): sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) @@ -25,8 +61,11 @@ def get_parser(): parser.add_argument("--dev", action="store_true", default=False) parser.add_argument("--debug", action="store_true", default=False) parser.add_argument("--system", action="store_true", default=False) + parser.add_argument("--parse-only", action="store_true", default=False) + parser.add_argument("--pipenv-site", metavar="pipenv_site_dir", action="store", + default=os.environ.get("PIPENV_SITE_DIR")) parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store", - default=os.environ.get("PIPENV_REQ_DIR")) + default=os.environ.get("PIPENV_REQ_DIR")) parser.add_argument("packages", nargs="*") return parser @@ -407,12 +446,39 @@ def clean_outdated(results, resolver, project, dev=False): return new_results -def _main(pre, clear, verbose, system, requirements_dir, dev, packages): - os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) - os.environ["PIP_PYTHON_PATH"] = str(sys.executable) +def parse_packages(packages, pre, clear, system, requirements_dir=None): + from pipenv.vendor.requirementslib.models.requirements import Requirement + from pipenv.vendor.vistir.contextmanagers import cd, temp_path + from pipenv.utils import parse_indexes + parsed_packages = [] + for package in packages: + indexes, trusted_hosts, line = parse_indexes(package) + line = " ".join(line) + pf = dict() + req = Requirement.from_line(line) + if not req.name: + with temp_path(), cd(req.req.setup_info.base_dir): + sys.path.insert(0, req.req.setup_info.base_dir) + req.req._setup_info.get_info() + req.update_name_from_path(req.req.setup_info.base_dir) + print(os.listdir(req.req.setup_info.base_dir)) + try: + name, entry = req.pipfile_entry + except Exception: + continue + else: + if name is not None and entry is not None: + pf[name] = entry + parsed_packages.append(pf) + print("RESULTS:") + if parsed_packages: + print(json.dumps(parsed_packages)) + else: + print(json.dumps([])) + +def resolve_packages(pre, clear, verbose, system, requirements_dir, packages): from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources - pypi_mirror_source = ( create_mirror_source(os.environ["PIPENV_PYPI_MIRROR"]) if "PIPENV_PYPI_MIRROR" in os.environ @@ -456,36 +522,46 @@ def _main(pre, clear, verbose, system, requirements_dir, dev, packages): print(json.dumps([])) -def main(): - _patch_path() - import warnings - from pipenv.vendor.vistir.compat import ResourceWarning - warnings.simplefilter("ignore", category=ResourceWarning) - import io - import six - if six.PY3: - import atexit - stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') - atexit.register(stdout_wrapper.close) - stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') - atexit.register(stderr_wrapper.close) - sys.stdout = stdout_wrapper - sys.stderr = stderr_wrapper +def _main(pre, clear, verbose, system, requirements_dir, packages, parse_only=False): + os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) + os.environ["PIP_PYTHON_PATH"] = str(sys.executable) + if parse_only: + parse_packages( + packages, + pre=pre, + clear=clear, + system=system, + requirements_dir=requirements_dir, + ) else: - from pipenv._compat import force_encoding - force_encoding() - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") - os.environ["PYTHONIOENCODING"] = str("utf-8") + resolve_packages(pre, clear, verbose, system, requirements_dir, packages) + + +def main(): parser = get_parser() parsed, remaining = parser.parse_known_args() - # sys.argv = remaining + _patch_path(pipenv_site=parsed.pipenv_site) + import warnings + from pipenv.vendor.vistir.compat import ResourceWarning + from pipenv.vendor.vistir.misc import get_wrapped_stream + warnings.simplefilter("ignore", category=ResourceWarning) + import six + if six.PY3: + stdout = sys.stdout.buffer + stderr = sys.stderr.buffer + else: + stdout = sys.stdout + stderr = sys.stderr + sys.stderr = get_wrapped_stream(stderr) + sys.stdout = get_wrapped_stream(stdout) + from pipenv.vendor import colorama + colorama.init() + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") + os.environ["PYTHONIOENCODING"] = str("utf-8") parsed = handle_parsed_args(parsed) _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, - parsed.requirements_dir, parsed.dev, parsed.packages) + parsed.requirements_dir, parsed.packages, parse_only=parsed.parse_only) if __name__ == "__main__": - _patch_path() - from pipenv.vendor import colorama - colorama.init() main() diff --git a/pipenv/utils.py b/pipenv/utils.py index d31ba843..406886fe 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -552,7 +552,7 @@ class Resolver(object): return cleaned_checksums def collect_hashes(self, ireq): - from requests import ConnectionError + from .vendor.requests import ConnectionError collected_hashes = [] if ireq in self.hashes: collected_hashes += list(self.hashes.get(ireq, [])) @@ -1313,7 +1313,7 @@ def get_canonical_names(packages): if not isinstance(packages, Sequence): if not isinstance(packages, six.string_types): return packages - packages = [packages,] + packages = [packages] return set([canonicalize_name(pkg) for pkg in packages if pkg]) @@ -1742,11 +1742,11 @@ def parse_indexes(line): ) parser.add_argument( "--extra-index-url", "--extra-index", - metavar="extra_indexes",action="append", + metavar="extra_indexes", action="append", ) parser.add_argument("--trusted-host", metavar="trusted_hosts", action="append") args, remainder = parser.parse_known_args(line.split()) - index = [] if not args.index else [args.index,] + index = [] if not args.index else [args.index] extra_indexes = [] if not args.extra_index_url else args.extra_index_url indexes = index + extra_indexes trusted_hosts = args.trusted_host if args.trusted_host else [] @@ -1792,3 +1792,12 @@ def is_url_equal(url, other_url): unparsed = parsed_url._replace(auth=None, query=None, fragment=None).url unparsed_other = parsed_other_url._replace(auth=None, query=None, fragment=None).url return unparsed == unparsed_other + + +def get_pipenv_dist(pkg="pipenv", pipenv_site=None): + from .resolver import find_site_path + pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + if pipenv_site is None: + pipenv_site = os.path.dirname(pipenv_libdir) + pipenv_dist, _ = find_site_path(pkg, site_dir=pipenv_site) + return pipenv_dist diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 5f2ac1db..b0b55d47 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -718,7 +718,7 @@ build-backend = "{1}" get_metadata(d, pkg_name=self.name, metadata_type=metadata_type) for d in metadata_dirs if os.path.exists(d) ] - metadata = next(iter(d for d in metadata if d), None) + metadata = next(iter(d for d in metadata if d is not None), None) if metadata is not None: self.populate_metadata(metadata) diff --git a/run-tests.bat b/run-tests.bat index d20af35f..a511fa45 100644 --- a/run-tests.bat +++ b/run-tests.bat @@ -1,7 +1,7 @@ rem imdisk -a -s 964515b -m R: -p "/FS:NTFS /Y" virtualenv R:\.venv -R:\.venv\Scripts\pip install -e . --upgrade --upgrade-strategy=only-if-needed +R:\.venv\Scripts\pip install -e .[test] --upgrade --upgrade-strategy=only-if-needed R:\.venv\Scripts\pipenv install --dev git submodule sync && git submodule update --init --recursive SET RAM_DISK=R: && R:\.venv\Scripts\pipenv run pytest -n auto -v tests --tap-stream > report.tap diff --git a/run-tests.sh b/run-tests.sh index 991c0df7..a99c0461 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -53,8 +53,8 @@ fi echo "Installing dependencies…" PIPENV_PYTHON=2.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --dev PIPENV_PYTHON=3.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --dev -PIPENV_PYTHON=2.7 python3 -m pipenv run pip install --upgrade -e . -PIPENV_PYTHON=3.7 python3 -m pipenv run pip install --upgrade -e . +PIPENV_PYTHON=2.7 python3 -m pipenv run pip install --upgrade -e .[test] +PIPENV_PYTHON=3.7 python3 -m pipenv run pip install --upgrade -e .[test] echo "$ git submodule sync && git submodule update --init --recursive" git submodule sync && git submodule update --init --recursive diff --git a/tests/integration/test_pipenv.py b/tests/integration/test_pipenv.py index deacc495..9db76d28 100644 --- a/tests/integration/test_pipenv.py +++ b/tests/integration/test_pipenv.py @@ -101,9 +101,13 @@ def test_directory_with_leading_dash(PipenvInstance): with mock.patch('pipenv.vendor.vistir.compat.mkdtemp', side_effect=mocked_mkdtemp): with temp_environ(), PipenvInstance(chdir=True) as p: - del os.environ['PIPENV_VENV_IN_PROJECT'] - p.pipenv('--python python') - venv_path = p.pipenv('--venv').out.strip() + if "PIPENV_VENV_IN_PROJECT" in os.environ: + del os.environ['PIPENV_VENV_IN_PROJECT'] + c = p.pipenv('--python python') + assert c.return_code == 0 + c = p.pipenv('--venv') + assert c.return_code == 0 + venv_path = c.out.strip() assert os.path.isdir(venv_path) # Manually clean up environment, since PipenvInstance assumes that # the virutalenv is in the project directory. diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 237fd483..487524c6 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -183,7 +183,7 @@ def test_run_in_virtualenv_with_global_context(PipenvInstance, pypi, virtualenv) assert c.return_code == 0 c = p.pipenv('run python -c "import click;print(click.__file__)"') assert c.return_code == 0 - assert c.out.strip().startswith(str(virtualenv)) + assert c.out.strip().startswith(virtualenv.as_posix()) c = p.pipenv("clean --dry-run") assert c.return_code == 0 assert "click" in c.out