From ffb46e42131f1706c7c6705b2686e07dda095108 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 25 Nov 2018 02:09:14 -0500 Subject: [PATCH 01/14] Fix direct url dependency resolution - Fix dependecy resolution for _all_ dependencies which are just direct URLs or files - Bypass pip-tools for non-editable requirements which can be resolved - Fixes #3148 Signed-off-by: Dan Ryan --- news/3148.bugfix.rst | 1 + pipenv/utils.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 news/3148.bugfix.rst diff --git a/news/3148.bugfix.rst b/news/3148.bugfix.rst new file mode 100644 index 00000000..1f0f4a62 --- /dev/null +++ b/news/3148.bugfix.rst @@ -0,0 +1 @@ +Fixed resolution of direct-url dependencies in ``setup.py`` files to respect ``PEP-508`` style URL dependencies. diff --git a/pipenv/utils.py b/pipenv/utils.py index 49975b16..1470d181 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -210,6 +210,27 @@ def prepare_pip_source_args(sources, pip_args=None): return pip_args +def resolve_separate(req): + """ + Resolve a requirement that the normal resolver can't + + This includes non-editable urls to zip or tarballs, non-editable paths, etc. + """ + + constraints = [] + if req.is_file_or_url and not req.is_vcs: + setup_info = req.run_requires() + requirements = [v for v in setup_info.get("requires", {}).values()] + for r in requirements: + if getattr(r, "url", None) and not getattr(r, "editable", False): + from .vendor.requirementslib.models.requirements import Requirement + requirement = Requirement.from_line("".join(str(r).split())) + constraints.extend(resolve_separate(requirement)) + continue + constraints.append(str(r)) + return constraints + + def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): from .vendor.requirementslib.models.requirements import Requirement constraints = [] @@ -222,6 +243,10 @@ def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): url = indexes[0] dep = " ".join(remainder) req = Requirement.from_line(dep) + if req.is_file_or_url and not req.is_vcs: + # TODO: This is a significant hack, should probably be reworked + constraints.extend(resolve_separate(req)) + continue constraints.append(req.constraint_line) if url: @@ -605,6 +630,12 @@ def venv_resolve_deps( results = [] pipfile_section = "dev_packages" if dev else "packages" lockfile_section = "develop" if dev else "default" + # TODO: We can only use all of the requirements here because we weed them out later + # in `get_resolver_metadata` via `resolve_separate` which uses requirementslib + # to handle the resolution of special case dependencies (including local paths and + # file urls). We should probably rework this at some point. + deps = project._pipfile.dev_requirements if dev else project._pipfile.requirements + vcs_deps = [r for r in deps if r.is_vcs] vcs_section = "vcs_{0}".format(pipfile_section) vcs_deps = getattr(project, vcs_section, {}) if not deps and not vcs_deps: @@ -615,6 +646,10 @@ def venv_resolve_deps( if not lockfile: lockfile = project._lockfile req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") + for dep in deps: + if dep.is_file_or_url and not dep.is_vcs: + name, entry = dep.pipfile_entry + lockfile[lockfile_section][name] = entry if vcs_deps: with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: vcs_reqs, vcs_lockfile = get_vcs_deps( @@ -627,6 +662,7 @@ def venv_resolve_deps( ) vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] lockfile[lockfile_section].update(vcs_lockfile) + deps = [r.as_line() for r in deps if not r.is_vcs] cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() From bd78dae6d4f7d4b86501714192c81f2d9b800bd0 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 25 Nov 2018 14:04:56 -0500 Subject: [PATCH 02/14] WIP continued, some fixes Signed-off-by: Dan Ryan --- pipenv/utils.py | 65 +++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 1470d181..6dbc87c7 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -217,7 +217,7 @@ def resolve_separate(req): This includes non-editable urls to zip or tarballs, non-editable paths, etc. """ - constraints = [] + constraints = set() if req.is_file_or_url and not req.is_vcs: setup_info = req.run_requires() requirements = [v for v in setup_info.get("requires", {}).values()] @@ -225,15 +225,16 @@ def resolve_separate(req): if getattr(r, "url", None) and not getattr(r, "editable", False): from .vendor.requirementslib.models.requirements import Requirement requirement = Requirement.from_line("".join(str(r).split())) - constraints.extend(resolve_separate(requirement)) + constraints |= resolve_separate(requirement) continue - constraints.append(str(r)) + constraints.add(str(r)) return constraints def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): from .vendor.requirementslib.models.requirements import Requirement - constraints = [] + constraints = set() + skipped = {} for dep in deps: if not dep: continue @@ -245,9 +246,11 @@ def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): req = Requirement.from_line(dep) if req.is_file_or_url and not req.is_vcs: # TODO: This is a significant hack, should probably be reworked - constraints.extend(resolve_separate(req)) + constraints |= resolve_separate(req) + name, entry = req.pipfile_entry + skipped[name] = entry continue - constraints.append(req.constraint_line) + constraints.add(req.constraint_line) if url: index_lookup[req.name] = project.get_source(url=url).get("name") @@ -256,7 +259,7 @@ def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): # eg pypiwin32 if req.markers: markers_lookup[req.name] = req.markers.replace('"', "'") - return constraints + return constraints, skipped class Resolver(object): @@ -492,17 +495,31 @@ def actually_resolve_deps( req_dir=None, ): from pipenv.vendor.vistir.path import create_tracked_tempdir + from pipenv.vendor.requirementslib.models.requirements import Requirement + from pipenv.vendor import pip_shims if not req_dir: req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") warning_list = [] with warnings.catch_warnings(record=True) as warning_list: - constraints = get_resolver_metadata( + constraints, skipped = get_resolver_metadata( deps, index_lookup, markers_lookup, project, sources, ) resolver = Resolver(constraints, req_dir, project, sources, clear=clear, pre=pre) resolved_tree = resolver.resolve() + for k, v in skipped.items(): + url = v.get("file") + req = Requirement.from_pipfile(k, v) + is_url = url and not url.startswith("file:") + path = v.get("path") + if not is_url and not path: + path = pip_shims.shims.url_to_path(url) + if is_url or (path and os.path.exists(path) and not os.path.isdir(path)): + existing = next(iter(req for req in resolved_tree if req.name == k), None) + if existing: + resolved_tree.remove(existing) + resolved_tree.add(req.as_ireq()) hashes = resolver.resolve_hashes() for warning in warning_list: @@ -630,14 +647,12 @@ def venv_resolve_deps( results = [] pipfile_section = "dev_packages" if dev else "packages" lockfile_section = "develop" if dev else "default" - # TODO: We can only use all of the requirements here because we weed them out later - # in `get_resolver_metadata` via `resolve_separate` which uses requirementslib - # to handle the resolution of special case dependencies (including local paths and - # file urls). We should probably rework this at some point. - deps = project._pipfile.dev_requirements if dev else project._pipfile.requirements - vcs_deps = [r for r in deps if r.is_vcs] vcs_section = "vcs_{0}".format(pipfile_section) - vcs_deps = getattr(project, vcs_section, {}) + if project.pipfile_exists: + deps = project._pipfile.dev_requirements if dev else project._pipfile.requirements + vcs_deps = [r for r in deps if r.is_vcs] + else: + vcs_deps = getattr(project, vcs_section, {}) if not deps and not vcs_deps: return {} @@ -714,8 +729,12 @@ def venv_resolve_deps( raise RuntimeError("There was a problem with locking.") lockfile[lockfile_section] = prepare_lockfile(results, pipfile, lockfile[lockfile_section]) for k, v in vcs_lockfile.items(): - if k in getattr(project, vcs_section, {}) or k not in lockfile[lockfile_section]: + if k in getattr(project, vcs_section, {}): lockfile[lockfile_section][k].update(v) + print(v) + else: + lockfile[lockfile_section][k] = v + print(v) def resolve_deps( @@ -795,6 +814,12 @@ def resolve_deps( collected_hashes = [] if result in hashes: collected_hashes = list(hashes.get(result)) + elif resolver._should_include_hashes(result): + try: + hash_map = resolver.resolver.resolve_hashes([result]) + collected_hashes = next(iter(hash_map.values()), []) + except (ValueError, KeyError, IndexError, ConnectionError): + pass elif any( "python.org" in source["url"] or "pypi.org" in source["url"] for source in sources @@ -819,14 +844,6 @@ def resolve_deps( crayons.red("Warning", bold=True), name ), err=True ) - # # Collect un-collectable hashes (should work with devpi). - # try: - # collected_hashes = collected_hashes + list( - # list(resolver.resolve_hashes([result]).items())[0][1] - # ) - # except (ValueError, KeyError, ConnectionError, IndexError): - # if verbose: - # print('Error generating hash for {}'.format(name)) req.hashes = sorted(set(collected_hashes)) name, _entry = req.pipfile_entry entry = {} From 9340ddb9623a1221c5f8c24303a52b0584c5f02d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 20 Feb 2019 02:38:02 -0500 Subject: [PATCH 03/14] Update tests to rectify issues Signed-off-by: Dan Ryan --- tests/integration/conftest.py | 5 +++++ tests/integration/test_install_uri.py | 4 ++-- tests/integration/test_project.py | 2 +- .../{Cerberus-1.2.tar.gz => cerberus-1.2.tar.gz} | Bin 4 files changed, 8 insertions(+), 3 deletions(-) rename tests/pypi/cerberus/{Cerberus-1.2.tar.gz => cerberus-1.2.tar.gz} (100%) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f96e6ec6..4330eea9 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,3 +1,4 @@ +# -*- coding=utf-8 -*- import json import os import sys @@ -80,6 +81,10 @@ def pytest_runtest_setup(item): pytest.skip('requires github ssh') if item.get_marker('needs_hg') is not None and not WE_HAVE_HG: pytest.skip('requires mercurial') + if item.get_marker('skip_py27_win') is not None and ( + sys.version_info[:2] <= (2, 7) and os.name == "nt" + ): + pytest.skip('must use python > 2.7 on windows') @pytest.fixture diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 664f30cc..e146d6d3 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -1,5 +1,4 @@ -import os - +# -*- coding=utf-8 -*- import pytest from flaky import flaky @@ -125,6 +124,7 @@ def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir): @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet +@pytest.mark.skip_py27_win @flaky def test_editable_vcs_install(PipenvInstance, pip_src_dir, pypi): with PipenvInstance(pypi=pypi) as p: diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 487524c6..57488258 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -175,7 +175,7 @@ def test_run_in_virtualenv_with_global_context(PipenvInstance, pypi, virtualenv) assert c.return_code == 0 assert 'Creating a virtualenv' not in c.err project = Project() - assert project.virtualenv_location == str(virtualenv) + assert project.virtualenv_location == virtualenv.as_posix() c = p.pipenv("run pip install click") assert c.return_code == 0 assert "Courtesy Notice" in c.err diff --git a/tests/pypi/cerberus/Cerberus-1.2.tar.gz b/tests/pypi/cerberus/cerberus-1.2.tar.gz similarity index 100% rename from tests/pypi/cerberus/Cerberus-1.2.tar.gz rename to tests/pypi/cerberus/cerberus-1.2.tar.gz From a47a26b87a14ece449c8b95967ebf9b9689c18ba Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 20 Feb 2019 11:51:15 -0500 Subject: [PATCH 04/14] Fix failing tests Signed-off-by: Dan Ryan --- tests/integration/test_install_basic.py | 4 ++-- tests/integration/test_install_twists.py | 2 +- tests/integration/test_install_uri.py | 9 +++++---- tests/integration/test_project.py | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index 2f91486e..3521ee5b 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -461,11 +461,11 @@ version = "*" extras = ["socks"] """.strip() f.write(contents) - c = p.pipenv("install plette[validation]") + c = p.pipenv("install plette") assert c.return_code == 0 with open(p.pipfile_path) as f: contents = f.read() assert "[packages.requests]" not in contents assert 'six = {version = "*"}' in contents assert 'requests = {version = "*"' in contents - assert 'plette = {' in contents + assert 'plette = "*"' in contents diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index af6dbd45..98236b8a 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -57,7 +57,7 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]} project.write_toml({"packages": {}, "dev-packages": {}}) c = p.pipenv("install {0}".format(line)) assert c.return_code == 0 - assert "testpipenv" in p.pipfile["packages"], "{0}\n{1}\n\n{2}\n\n{3}".format(p.pipfile, Path(p.pipfile_path).read_text(), Path(os.getcwd()).joinpath("setup.py").read_text(), Path(os.path.join(os.getcwd(), "testpipenv.egg-info/PKG-INFO")).read_text()) + assert "testpipenv" in p.pipfile["packages"] assert p.pipfile["packages"]["testpipenv"]["path"] == "." assert p.pipfile["packages"]["testpipenv"]["extras"] == ["dev"] assert "six" in p.lockfile["default"] diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index e146d6d3..b0c1e51c 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -124,7 +124,6 @@ def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir): @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -@pytest.mark.skip_py27_win @flaky def test_editable_vcs_install(PipenvInstance, pip_src_dir, pypi): with PipenvInstance(pypi=pypi) as p: @@ -239,13 +238,15 @@ def test_get_vcs_refs(PipenvInstance, pip_src_dir): @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet +@pytest.mark.skip_py27_win def test_vcs_entry_supersedes_non_vcs(PipenvInstance, pip_src_dir): """See issue #2181 -- non-editable VCS dep was specified, but not showing up in the lockfile -- due to not running pip install before locking and not locking the resolution graph of non-editable vcs dependencies. """ with PipenvInstance(chdir=True) as p: - pyinstaller_path = p._pipfile.get_fixture_path("git/pyinstaller") + # pyinstaller_path = p._pipfile.get_fixture_path("git/pyinstaller") + pyinstaller_uri = "https://github.com/pyinstaller/pyinstaller.git" with open(p.pipfile_path, "w") as f: f.write( """ @@ -257,7 +258,7 @@ name = "pypi" [packages] PyUpdater = "*" PyInstaller = {{ref = "develop", git = "{0}"}} - """.format(pyinstaller_path.as_uri()).strip() + """.format(pyinstaller_uri).strip() ) c = p.pipenv("install") assert c.return_code == 0 @@ -268,7 +269,7 @@ PyInstaller = {{ref = "develop", git = "{0}"}} assert p.lockfile["default"]["pyinstaller"].get("ref") is not None assert ( p.lockfile["default"]["pyinstaller"]["git"] - == pyinstaller_path.as_uri() + == pyinstaller_uri ) diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 57488258..c3eb4096 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -183,7 +183,7 @@ def test_run_in_virtualenv_with_global_context(PipenvInstance, pypi, virtualenv) assert c.return_code == 0 c = p.pipenv('run python -c "import click;print(click.__file__)"') assert c.return_code == 0 - assert c.out.strip().startswith(virtualenv.as_posix()) + assert c.out.strip().startswith(str(virtualenv)) c = p.pipenv("clean --dry-run") assert c.return_code == 0 assert "click" in c.out From 16a249f64ff96eaf31b569c6e213eaab1e4f6f11 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 20 Feb 2019 17:52:04 -0500 Subject: [PATCH 05/14] Fix installed package discovery on windows - Fix prefix formatting - Add posix formatting function Signed-off-by: Dan Ryan --- pipenv/environment.py | 15 ++++++++------- pipenv/project.py | 8 ++------ pipenv/utils.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index 5e2ec3f5..d06bae8e 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -19,7 +19,7 @@ import pipenv from cached_property import cached_property -from .utils import normalize_path +from .utils import normalize_path, make_posix BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path) @@ -145,7 +145,7 @@ class Environment(object): 'stdlib': '/home/hawk/.pyenv/versions/3.7.1/lib/python3.7'} """ - prefix = self.prefix.as_posix() + prefix = make_posix(self.prefix.as_posix()) install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' paths = get_paths(install_scheme, vars={ 'base': prefix, @@ -154,8 +154,8 @@ class Environment(object): paths["PATH"] = paths["scripts"] + os.pathsep + os.defpath if "prefix" not in paths: paths["prefix"] = prefix - purelib = get_python_lib(plat_specific=0, prefix=prefix) - platlib = get_python_lib(plat_specific=1, prefix=prefix) + purelib = make_posix(get_python_lib(plat_specific=0, prefix=prefix)) + platlib = make_posix(get_python_lib(plat_specific=1, prefix=prefix)) if purelib == platlib: lib_dirs = purelib else: @@ -272,13 +272,14 @@ class Environment(object): """Determine whether the supplied distribution is in the environment.""" from .project import _normalized prefixes = [ - _normalized(prefix) for prefix in self.base_paths["libdirs"] - if _normalized(self.prefix).startswith(_normalized(prefix)) + _normalized(prefix) for prefix in self.base_paths["libdirs"].split(os.pathsep) + if _normalized(prefix).startswith(_normalized(self.prefix.as_posix())) ] location = self.locate_dist(dist) if not location: return False - return any(_normalized(location).startswith(prefix) for prefix in prefixes) + location = _normalized(make_posix(location)) + return any(location.startswith(prefix) for prefix in prefixes) def get_installed_packages(self): """Returns all of the installed packages in a given environment""" diff --git a/pipenv/project.py b/pipenv/project.py index 68ec4666..7d359a8e 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -525,8 +525,8 @@ class Project(object): if not os.path.exists(self.path_to("setup.py")): if not build_system or not build_system.get("requires"): build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8.0", "wheel"], + "build-backend": "setuptools.build_meta:__legacy__", } self._build_system = build_system @@ -603,10 +603,8 @@ class Project(object): def _get_editable_packages(self, dev=False): section = "dev-packages" if dev else "packages" - # section = "{0}-editable".format(section) packages = { k: v - # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() if is_editable(k) or is_editable(v) } @@ -615,10 +613,8 @@ class Project(object): def _get_vcs_packages(self, dev=False): from pipenv.vendor.requirementslib.utils import is_vcs section = "dev-packages" if dev else "packages" - # section = "{0}-vcs".format(section) packages = { k: v - # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() if is_vcs(v) or is_vcs(k) } diff --git a/pipenv/utils.py b/pipenv/utils.py index d7093495..f2f7704e 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -3,6 +3,7 @@ import contextlib import errno import logging import os +import posixpath import re import shutil import stat @@ -1769,7 +1770,22 @@ def add_to_set(original_set, element): def is_url_equal(url, other_url): # type: (str, str) -> bool - """Compare two urls by scheme, host, and path, ignoring auth""" + """ + Compare two urls by scheme, host, and path, ignoring auth + + :param str url: The initial URL to compare + :param str url: Second url to compare to the first + :return: Whether the URLs are equal without **auth**, **query**, and **fragment** + :rtype: bool + + >>> is_url_equal("https://user:pass@mydomain.com/some/path?some_query", + "https://user2:pass2@mydomain.com/some/path") + True + + >>> is_url_equal("https://user:pass@mydomain.com/some/path?some_query", + "https://mydomain.com/some?some_query") + False + """ if not isinstance(url, six.string_types): raise TypeError("Expected string for url, received {0!r}".format(url)) if not isinstance(other_url, six.string_types): @@ -1779,3 +1795,28 @@ def is_url_equal(url, other_url): unparsed = parsed_url._replace(auth=None, query=None, fragment=None).url unparsed_other = parsed_other_url._replace(auth=None, query=None, fragment=None).url return unparsed == unparsed_other + + +@lru_cache() +def make_posix(path): + # type: (str) -> str + """ + Convert a path with possible windows-style separators to a posix-style path + (with **/** separators instead of **\\** separators). + + :param Text path: A path to convert. + :return: A converted posix-style path + :rtype: Text + + >>> make_posix("c:/users/user/venvs/some_venv\\Lib\\site-packages") + "c:/users/user/venvs/some_venv/Lib/site-packages" + + >>> make_posix("c:\\users\\user\\venvs\\some_venv") + "c:/users/user/venvs/some_venv" + """ + if not isinstance(path, six.string_types): + raise TypeError("Expected a string for path, received {0!r}...".format(path)) + separated = normalize_path(path).split(os.path.sep) + if isinstance(separated, (list, tuple)): + path = posixpath.join(*separated) + return path From cd2cbd4e1ad664cbc4b54ed50e0da41b65016b38 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 20 Feb 2019 18:20:51 -0500 Subject: [PATCH 06/14] Avoid using `which` in cross platform tests Signed-off-by: Dan Ryan --- tests/integration/test_project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index c3eb4096..d193fc7d 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -171,7 +171,7 @@ def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpd @pytest.mark.virtualenv def test_run_in_virtualenv_with_global_context(PipenvInstance, pypi, virtualenv): with PipenvInstance(chdir=True, pypi=pypi, venv_root=virtualenv.as_posix(), ignore_virtualenvs=False, venv_in_project=False) as p: - c = p.pipenv('run which pip') + c = p.pipenv('run pip freeze') assert c.return_code == 0 assert 'Creating a virtualenv' not in c.err project = Project() @@ -193,7 +193,7 @@ def test_run_in_virtualenv_with_global_context(PipenvInstance, pypi, virtualenv) @pytest.mark.virtualenv def test_run_in_virtualenv(PipenvInstance, pypi): with PipenvInstance(chdir=True, pypi=pypi) as p: - c = p.pipenv('run which pip') + c = p.pipenv('run pip freeze') assert c.return_code == 0 assert 'Creating a virtualenv' in c.err project = Project() From 3bc48f034fc79357324b7b71322af152aa6ec483 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 21 Feb 2019 22:18:00 -0500 Subject: [PATCH 07/14] Fix venv with leading dash test on windows and clean up tests Signed-off-by: Dan Ryan --- pipenv/_compat.py | 3 -- pipenv/environment.py | 6 +++- pipenv/project.py | 5 +-- pipenv/utils.py | 2 +- pipenv/vendor/requirementslib/models/utils.py | 8 ++--- tests/integration/conftest.py | 19 ++++++++-- tests/integration/test_pipenv.py | 36 +++++++------------ 7 files changed, 42 insertions(+), 37 deletions(-) diff --git a/pipenv/_compat.py b/pipenv/_compat.py index e8ef16e5..caba80fe 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -5,9 +5,6 @@ Exposes a standard API that enables compatibility across python versions, operating systems, etc. """ -import functools -import importlib -import io import os import sys import warnings diff --git a/pipenv/environment.py b/pipenv/environment.py index d06bae8e..2bfc5334 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -11,6 +11,7 @@ import sys from distutils.sysconfig import get_python_lib from sysconfig import get_paths +import itertools import pkg_resources import six import vistir @@ -244,7 +245,10 @@ class Environment(object): """ pkg_resources = self.safe_import("pkg_resources") - return pkg_resources.find_distributions(self.paths["libdirs"]) + libdirs = self.paths["libdirs"].split(os.pathsep) + dists = (pkg_resources.find_distributions(libdir) for libdir in libdirs) + for dist in itertools.chain.from_iterable(dists): + yield dist def find_egg(self, egg_dist): """Find an egg by name in the given environment""" diff --git a/pipenv/project.py b/pipenv/project.py index 7d359a8e..c71c40ea 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -29,6 +29,7 @@ from .environments import ( PIPENV_PIPFILE, PIPENV_PYTHON, PIPENV_TEST_INDEX, PIPENV_VENV_IN_PROJECT, is_in_virtualenv ) +from .vendor.requirementslib.models.utils import get_default_pyproject_backend from .utils import ( cleanup_toml, convert_toml_outline_tables, find_requirements, get_canonical_names, get_url_name, get_workon_home, is_editable, @@ -526,7 +527,7 @@ class Project(object): if not build_system or not build_system.get("requires"): build_system = { "requires": ["setuptools>=40.8.0", "wheel"], - "build-backend": "setuptools.build_meta:__legacy__", + "build-backend": get_default_pyproject_backend(), } self._build_system = build_system @@ -536,7 +537,7 @@ class Project(object): @property def build_backend(self): - return self._build_system.get("build-backend", None) + return self._build_system.get("build-backend", get_default_pyproject_backend()) @property def settings(self): diff --git a/pipenv/utils.py b/pipenv/utils.py index f2f7704e..199e2b41 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -36,7 +36,7 @@ from .vendor.urllib3 import util as urllib3_util if environments.MYPY_RUNNING: - from typing import Tuple, Dict, Any, List, Union, Optional + from typing import Tuple, Dict, Any, List, Union, Optional, Text from .vendor.requirementslib.models.requirements import Requirement, Line from .project import Project diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index c9852a7b..8ce441e2 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -366,7 +366,7 @@ def get_pyproject(path): if not pp_toml.exists(): if not setup_py.exists(): return None - requires = ["setuptools>=40.6", "wheel"] + requires = ["setuptools>=40.8", "wheel"] backend = get_default_pyproject_backend() else: pyproject_data = {} @@ -375,10 +375,10 @@ def get_pyproject(path): build_system = pyproject_data.get("build-system", None) if build_system is None: if setup_py.exists(): - requires = ["setuptools>=40.6", "wheel"] + requires = ["setuptools>=40.8", "wheel"] backend = get_default_pyproject_backend() else: - requires = ["setuptools>=40.6", "wheel"] + requires = ["setuptools>=40.8", "wheel"] backend = get_default_pyproject_backend() build_system = { "requires": requires, @@ -386,7 +386,7 @@ def get_pyproject(path): } pyproject_data["build_system"] = build_system else: - requires = build_system.get("requires", ["setuptools>=40.6", "wheel"]) + requires = build_system.get("requires", ["setuptools>=40.8", "wheel"]) backend = build_system.get("build-backend", get_default_pyproject_backend()) return (requires, backend) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 4330eea9..1f150b75 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -119,6 +119,8 @@ def isolate(pathlib_tmpdir): os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org") mkdir_p(os.path.join(home_dir, ".virtualenvs")) os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) + os.environ["HOME"] = home_dir + mkdir_p(os.path.join(home_dir, "projects")) # Ignore PIPENV_ACTIVE so that it works as under a bare environment. os.environ.pop("PIPENV_ACTIVE", None) os.environ.pop("VIRTUAL_ENV", None) @@ -194,7 +196,7 @@ class _PipenvInstance(object): """An instance of a Pipenv Project...""" def __init__( self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None, - venv_root=None, ignore_virtualenvs=True, venv_in_project=True + venv_root=None, ignore_virtualenvs=True, venv_in_project=True, name=None ): self.pypi = pypi if ignore_virtualenvs: @@ -208,13 +210,24 @@ class _PipenvInstance(object): self.original_dir = os.path.abspath(os.curdir) path = path if path else os.environ.get("PIPENV_PROJECT_DIR", None) + if name is not None: + path = Path(os.environ["HOME"]) / "projects" / name + path.mkdir(exist_ok=True) if not path: - self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + if isinstance(path, TemporaryDirectory): + self._path = path path = Path(self._path.name) try: self.path = str(path.resolve()) except OSError: self.path = str(path.absolute()) + elif isinstance(path, Path): + self._path = path + try: + self.path = str(path.resolve()) + except OSError: + self.path = str(path.absolute()) else: self._path = path self.path = path @@ -245,7 +258,7 @@ class _PipenvInstance(object): if self.chdir: os.chdir(self.original_dir) self.path = None - if self._path: + if self._path and getattr(self._path, "cleanup", None): try: self._path.cleanup() except OSError as e: diff --git a/tests/integration/test_pipenv.py b/tests/integration/test_pipenv.py index 9db76d28..27d31799 100644 --- a/tests/integration/test_pipenv.py +++ b/tests/integration/test_pipenv.py @@ -5,12 +5,8 @@ XXX: Try our best to reduce tests in this file. import os -from tempfile import mkdtemp - -import mock import pytest -from pipenv._compat import Path from pipenv.project import Project from pipenv.utils import temp_environ from pipenv.vendor import delegator @@ -93,22 +89,16 @@ def test_proper_names_unamanged_virtualenv(PipenvInstance, pypi): @pytest.mark.cli -def test_directory_with_leading_dash(PipenvInstance): - def mocked_mkdtemp(suffix, prefix, dir): - if suffix == '-project': - prefix = '-dir-with-leading-dash' - return mkdtemp(suffix, prefix, dir) - - with mock.patch('pipenv.vendor.vistir.compat.mkdtemp', side_effect=mocked_mkdtemp): - with temp_environ(), PipenvInstance(chdir=True) as p: - if "PIPENV_VENV_IN_PROJECT" in os.environ: - del os.environ['PIPENV_VENV_IN_PROJECT'] - c = p.pipenv('--python python') - assert c.return_code == 0 - c = p.pipenv('--venv') - assert c.return_code == 0 - venv_path = c.out.strip() - assert os.path.isdir(venv_path) - # Manually clean up environment, since PipenvInstance assumes that - # the virutalenv is in the project directory. - p.pipenv('--rm') +def test_directory_with_leading_dash(monkeypatch, PipenvInstance): + with temp_environ(), PipenvInstance(chdir=True, venv_in_project=False, name="-project-with-dash") as p: + if "PIPENV_VENV_IN_PROJECT" in os.environ: + del os.environ['PIPENV_VENV_IN_PROJECT'] + c = p.pipenv('run pip freeze') + assert c.return_code == 0 + c = p.pipenv('--venv') + assert c.return_code == 0 + venv_path = c.out.strip() + assert os.path.isdir(venv_path) + # Manually clean up environment, since PipenvInstance assumes that + # the virutalenv is in the project directory. + p.pipenv('--rm') From 7f7550764bbd7306ba8efcaa01ab8d8fdfba22e4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 22 Feb 2019 02:26:04 -0500 Subject: [PATCH 08/14] Clean up path checks in environment Signed-off-by: Dan Ryan --- pipenv/environment.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index 2bfc5334..9c13611e 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -14,11 +14,11 @@ from sysconfig import get_paths import itertools import pkg_resources import six -import vistir import pipenv -from cached_property import cached_property +from .vendor.cached_property import cached_property +from .vendor import vistir from .utils import normalize_path, make_posix @@ -245,7 +245,7 @@ class Environment(object): """ pkg_resources = self.safe_import("pkg_resources") - libdirs = self.paths["libdirs"].split(os.pathsep) + libdirs = self.base_paths["libdirs"].split(os.pathsep) dists = (pkg_resources.find_distributions(libdir) for libdir in libdirs) for dist in itertools.chain.from_iterable(dists): yield dist @@ -296,7 +296,7 @@ class Environment(object): @contextlib.contextmanager def get_finder(self, pre=False): - from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder + from .vendor.pip_shims.shims import Command, cmdoptions, index_group, PackageFinder from .environments import PIPENV_CACHE_DIR index_urls = [source.get("url") for source in self.sources] @@ -617,10 +617,7 @@ class Environment(object): monkey_patch.activate() pip_shims = self.safe_import("pip_shims") pathset_base = pip_shims.UninstallPathSet - import recursive_monkey_patch - recursive_monkey_patch.monkey_patch( - PatchedUninstaller, pathset_base - ) + pathset_base._permitted = PatchedUninstaller._permitted dist = next( iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), None From 3f3e61ecf1922408cbc129b28cbe86cb8472f3db Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 22 Feb 2019 12:12:45 -0500 Subject: [PATCH 09/14] Fix path reformatting for posix python path Signed-off-by: Dan Ryan --- pipenv/environment.py | 2 +- pipenv/project.py | 9 +++++---- pipenv/utils.py | 13 ++++++++----- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index 9c13611e..bf4df591 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -177,7 +177,7 @@ class Environment(object): @property def python(self): """Path to the environment python""" - py = vistir.compat.Path(self.base_paths["scripts"]).joinpath("python").as_posix() + py = vistir.compat.Path(self.base_paths["scripts"]).joinpath("python").absolute().as_posix() if not py: return vistir.compat.Path(sys.executable).as_posix() return py diff --git a/pipenv/project.py b/pipenv/project.py index c71c40ea..af155cc6 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -17,10 +17,10 @@ import vistir from first import first -import pipfile -import pipfile.api +from .vendor import pipfile +from .vendor.pipfile import api as pipfile_api -from cached_property import cached_property +from .vendor.cached_property import cached_property from .cmdparse import Script from .environment import Environment @@ -117,7 +117,8 @@ else: u"name": u"pypi", } -pipfile.api.DEFAULT_SOURCE = DEFAULT_SOURCE +pipfile_api.DEFAULT_SOURCE = DEFAULT_SOURCE +pipfile.api = pipfile_api class SourceNotFound(KeyError): diff --git a/pipenv/utils.py b/pipenv/utils.py index 199e2b41..b1419474 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -26,8 +26,8 @@ from six.moves.urllib.parse import urlparse from vistir.compat import ResourceWarning, lru_cache from vistir.misc import fs_str -import crayons -import parse +from .vendor import crayons +from .vendor import parse from . import environments from .exceptions import PipenvUsageError @@ -281,7 +281,7 @@ class Resolver(object): @staticmethod @lru_cache() def _get_pip_command(): - from pip_shims.shims import Command + from .vendor.pip_shims.shims import Command class PipCommand(Command): """Needed for pip-tools.""" @@ -1373,7 +1373,7 @@ def temp_path(): def load_path(python): from ._compat import Path - import delegator + from .vendor import delegator import json python = Path(python).as_posix() json_dump_commmand = '"import json, sys; print(json.dumps(sys.path));"' @@ -1487,7 +1487,7 @@ def handle_remove_readonly(func, path, exc): warnings.warn(default_warning_message.format(path), ResourceWarning) return - raise + raise exc def escape_cmd(cmd): @@ -1816,7 +1816,10 @@ def make_posix(path): """ if not isinstance(path, six.string_types): raise TypeError("Expected a string for path, received {0!r}...".format(path)) + starts_with_sep = path.startswith(os.path.sep) separated = normalize_path(path).split(os.path.sep) if isinstance(separated, (list, tuple)): path = posixpath.join(*separated) + if starts_with_sep: + path = "/{0}".format(path) return path From 565850d4fd11fc96d4ebb377a81764e1507b6442 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 22 Feb 2019 17:29:36 -0500 Subject: [PATCH 10/14] Fix broken imports Signed-off-by: Dan Ryan --- pipenv/environment.py | 2 +- pipenv/project.py | 7 +++---- pipenv/utils.py | 12 ++++++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index bf4df591..267f8fe2 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -18,7 +18,7 @@ import six import pipenv from .vendor.cached_property import cached_property -from .vendor import vistir +import vistir from .utils import normalize_path, make_posix diff --git a/pipenv/project.py b/pipenv/project.py index af155cc6..448a431a 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -17,8 +17,8 @@ import vistir from first import first -from .vendor import pipfile -from .vendor.pipfile import api as pipfile_api +import pipfile +import pipfile.api from .vendor.cached_property import cached_property @@ -117,8 +117,7 @@ else: u"name": u"pypi", } -pipfile_api.DEFAULT_SOURCE = DEFAULT_SOURCE -pipfile.api = pipfile_api +pipfile.api.DEFAULT_SOURCE = DEFAULT_SOURCE class SourceNotFound(KeyError): diff --git a/pipenv/utils.py b/pipenv/utils.py index b1419474..2f61d72b 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -23,11 +23,11 @@ six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # noqa from six.moves import Mapping, Sequence, Set from six.moves.urllib.parse import urlparse -from vistir.compat import ResourceWarning, lru_cache -from vistir.misc import fs_str +from .vendor.vistir.compat import ResourceWarning, lru_cache +from .vendor.vistir.misc import fs_str -from .vendor import crayons -from .vendor import parse +import crayons +import parse from . import environments from .exceptions import PipenvUsageError @@ -768,7 +768,7 @@ def create_spinner(text, nospin=None, spinner_name=None): def resolve(cmd, sp): - from .vendor import delegator + import delegator from .cmdparse import Script from .vendor.pexpect.exceptions import EOF, TIMEOUT from .vendor.vistir.compat import to_native_string @@ -1373,7 +1373,7 @@ def temp_path(): def load_path(python): from ._compat import Path - from .vendor import delegator + import delegator import json python = Path(python).as_posix() json_dump_commmand = '"import json, sys; print(json.dumps(sys.path));"' From 018e6fe29f065809b19d1ff3f82c1397743ac71b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 22 Feb 2019 21:07:42 -0500 Subject: [PATCH 11/14] Fix pipenv dist discovery Signed-off-by: Dan Ryan --- pipenv/environment.py | 8 ++++++-- pipenv/project.py | 8 ++++++-- pipenv/utils.py | 9 +++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index 5e2ec3f5..0140a0a4 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -92,12 +92,16 @@ class Environment(object): deps |= cls.resolve_dist(dist, working_set) return deps - def add_dist(self, dist_name): - dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + def extend_dists(self, dist): extras = self.resolve_dist(dist, self.base_working_set) + self.extra_dists.append(dist) if extras: self.extra_dists.extend(extras) + def add_dist(self, dist_name): + dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + self.extend_dists(dist) + @cached_property def python_version(self): with self.activated(): diff --git a/pipenv/project.py b/pipenv/project.py index 68ec4666..55921414 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -34,7 +34,7 @@ from .utils import ( get_canonical_names, get_url_name, get_workon_home, is_editable, is_installable_file, is_star, is_valid_url, is_virtual_environment, looks_like_dir, normalize_drive, pep423_name, proper_case, python_version, - safe_expandvars + safe_expandvars, get_pipenv_dist ) @@ -340,7 +340,11 @@ class Project(object): prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile, project=self ) - self._environment.add_dist("pipenv") + pipenv_dist = get_pipenv_dist(pkg="pipenv") + if pipenv_dist: + self._environment.extend_dists(pipenv_dist) + else: + self._environment.add_dist("pipenv") return self._environment def get_outdated_packages(self): diff --git a/pipenv/utils.py b/pipenv/utils.py index d7093495..93896aba 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1779,3 +1779,12 @@ def is_url_equal(url, other_url): unparsed = parsed_url._replace(auth=None, query=None, fragment=None).url unparsed_other = parsed_other_url._replace(auth=None, query=None, fragment=None).url return unparsed == unparsed_other + + +def get_pipenv_dist(pkg="pipenv", pipenv_site=None): + from .resolver import find_site_path + pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + if pipenv_site is None: + pipenv_site = os.path.dirname(pipenv_libdir) + pipenv_dist, _ = find_site_path(pkg, site_dir=pipenv_site) + return pipenv_dist From 090619a6a1f02994dfa1a04384aab8966df9f541 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 23 Feb 2019 13:11:32 -0500 Subject: [PATCH 12/14] Fix virtualenv tempdir lifespan Signed-off-by: Dan Ryan --- tests/integration/conftest.py | 92 +++++++++++++++++++++++++------- tests/integration/test_pipenv.py | 27 +++++----- 2 files changed, 87 insertions(+), 32 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1f150b75..929f3baf 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -8,7 +8,7 @@ import pytest from vistir.compat import ResourceWarning, fs_str from vistir.contextmanagers import temp_environ -from vistir.path import mkdir_p +from vistir.path import mkdir_p, create_tracked_tempdir from pipenv._compat import Path, TemporaryDirectory from pipenv.exceptions import VirtualenvActivationException @@ -96,9 +96,33 @@ def pathlib_tmpdir(request, tmpdir): pass +def _create_tracked_dir(): + tmp_location = os.environ.get("TEMP", os.environ.get("TMP")) + temp_args = {"prefix": "pipenv-", "suffix": "-test"} + if tmp_location is not None: + temp_args["dir"] = tmp_location + temp_path = create_tracked_tempdir(**temp_args) + return temp_path + + +@pytest.fixture +def vistir_tmpdir(): + temp_path = _create_tracked_dir() + yield Path(temp_path) + + +@pytest.fixture(name='create_tmpdir') +def vistir_tmpdir_factory(): + + def create_tmpdir(): + return Path(_create_tracked_dir()) + + yield create_tmpdir + + # Borrowed from pip's test runner filesystem isolation @pytest.fixture(autouse=True) -def isolate(pathlib_tmpdir): +def isolate(create_tmpdir): """ Isolate our tests so that things like global configuration files and the like do not affect our test results. @@ -107,7 +131,7 @@ def isolate(pathlib_tmpdir): """ # Create a directory to use as our home location. - home_dir = os.path.join(str(pathlib_tmpdir), "home") + home_dir = os.path.join(str(create_tmpdir()), "home") os.makedirs(home_dir) mkdir_p(os.path.join(home_dir, ".config", "git")) with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: @@ -117,8 +141,8 @@ def isolate(pathlib_tmpdir): os.environ["GIT_CONFIG_NOSYSTEM"] = fs_str("1") os.environ["GIT_AUTHOR_NAME"] = fs_str("pipenv") os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org") - mkdir_p(os.path.join(home_dir, ".virtualenvs")) - os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) + workon_home = create_tmpdir() + os.environ["WORKON_HOME"] = fs_str(str(workon_home)) os.environ["HOME"] = home_dir mkdir_p(os.path.join(home_dir, "projects")) # Ignore PIPENV_ACTIVE so that it works as under a bare environment. @@ -340,20 +364,50 @@ def testsroot(): return TESTS_ROOT -@pytest.fixture() -def virtualenv(pathlib_tmpdir): - virtualenv_path = pathlib_tmpdir / "venv" - with temp_environ(): - c = delegator.run("virtualenv {}".format(virtualenv_path), block=True) +class VirtualEnv(object): + def __init__(self, name="venv", base_dir=None): + if base_dir is None: + base_dir = Path(_create_tracked_dir()) + self.base_dir = base_dir + self.name = name + self.path = base_dir / name + + def __enter__(self): + self.create() + return self.activate() + + def __exit__(self, *args, **kwargs): + pass + + def create(self): + python = Path(sys.executable).as_posix() + cmd = "{0} -m virtualenv {1}".format(python, self.path.as_posix()) + c = delegator.run(cmd, block=True) assert c.return_code == 0 - for name in ("bin", "Scripts"): - activate_this = virtualenv_path / name / "activate_this.py" - if activate_this.exists(): - with open(str(activate_this)) as f: - code = compile(f.read(), str(activate_this), "exec") - exec(code, dict(__file__=str(activate_this))) - break + + def activate(self): + script_paths = [ + self.path.joinpath(name).joinpath("activate_this.py") + for name in ("bin", "Scripts") + ] + activate_this = next(iter(path for path in script_paths if path.exists()), None) + if activate_this is not None: + with open(str(activate_this)) as f: + code = compile(f.read(), str(activate_this), "exec") + exec(code, dict(__file__=str(activate_this))) + os.environ["VIRTUAL_ENV"] = str(self.path) + return self.path else: raise VirtualenvActivationException("Can't find the activate_this.py script.") - os.environ["VIRTUAL_ENV"] = str(virtualenv_path) - yield virtualenv_path + + +@pytest.fixture() +def virtualenv(vistir_tmpdir): + with temp_environ(): + venv = VirtualEnv(base_dir=vistir_tmpdir) + yield venv.activate() + + +@pytest.fixture() +def raw_venv(): + yield VirtualEnv diff --git a/tests/integration/test_pipenv.py b/tests/integration/test_pipenv.py index 27d31799..a5f10216 100644 --- a/tests/integration/test_pipenv.py +++ b/tests/integration/test_pipenv.py @@ -89,16 +89,17 @@ def test_proper_names_unamanged_virtualenv(PipenvInstance, pypi): @pytest.mark.cli -def test_directory_with_leading_dash(monkeypatch, PipenvInstance): - with temp_environ(), PipenvInstance(chdir=True, venv_in_project=False, name="-project-with-dash") as p: - if "PIPENV_VENV_IN_PROJECT" in os.environ: - del os.environ['PIPENV_VENV_IN_PROJECT'] - c = p.pipenv('run pip freeze') - assert c.return_code == 0 - c = p.pipenv('--venv') - assert c.return_code == 0 - venv_path = c.out.strip() - assert os.path.isdir(venv_path) - # Manually clean up environment, since PipenvInstance assumes that - # the virutalenv is in the project directory. - p.pipenv('--rm') +def test_directory_with_leading_dash(raw_venv, PipenvInstance): + with temp_environ(): # , raw_venv(name="-venv-with-dash") as venv: + with PipenvInstance(chdir=True, venv_in_project=False, name="-project-with-dash") as p: # venv_root=venv.parent.as_posix(), ignore_virtualenvs=False) as p: + if "PIPENV_VENV_IN_PROJECT" in os.environ: + del os.environ['PIPENV_VENV_IN_PROJECT'] + c = p.pipenv('run pip freeze') + assert c.return_code == 0 + c = p.pipenv('--venv') + assert c.return_code == 0 + venv_path = c.out.strip() + assert os.path.isdir(venv_path) + # Manually clean up environment, since PipenvInstance assumes that + # the virutalenv is in the project directory. + p.pipenv('--rm') From b0ba9a8766dc2c3c18af458b5830ed1847fea3f6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 23 Feb 2019 14:23:18 -0500 Subject: [PATCH 13/14] Update syntax for context manager virtualenv test class Signed-off-by: Dan Ryan --- tests/integration/conftest.py | 17 +++++++---------- tests/integration/test_pipenv.py | 4 ++-- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 929f3baf..e400fc06 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -373,11 +373,12 @@ class VirtualEnv(object): self.path = base_dir / name def __enter__(self): + self._old_environ = os.environ.copy() self.create() return self.activate() def __exit__(self, *args, **kwargs): - pass + os.environ = self._old_environ def create(self): python = Path(sys.executable).as_posix() @@ -386,12 +387,9 @@ class VirtualEnv(object): assert c.return_code == 0 def activate(self): - script_paths = [ - self.path.joinpath(name).joinpath("activate_this.py") - for name in ("bin", "Scripts") - ] - activate_this = next(iter(path for path in script_paths if path.exists()), None) - if activate_this is not None: + script_path = "Scripts" if os.name == "nt" else "bin" + activate_this = self.path / script_path / "activate_this.py" + if activate_this.exists(): with open(str(activate_this)) as f: code = compile(f.read(), str(activate_this), "exec") exec(code, dict(__file__=str(activate_this))) @@ -403,9 +401,8 @@ class VirtualEnv(object): @pytest.fixture() def virtualenv(vistir_tmpdir): - with temp_environ(): - venv = VirtualEnv(base_dir=vistir_tmpdir) - yield venv.activate() + with temp_environ(), VirtualEnv(base_dir=vistir_tmpdir) as venv: + yield venv @pytest.fixture() diff --git a/tests/integration/test_pipenv.py b/tests/integration/test_pipenv.py index a5f10216..ef8c23f2 100644 --- a/tests/integration/test_pipenv.py +++ b/tests/integration/test_pipenv.py @@ -90,8 +90,8 @@ def test_proper_names_unamanged_virtualenv(PipenvInstance, pypi): @pytest.mark.cli def test_directory_with_leading_dash(raw_venv, PipenvInstance): - with temp_environ(): # , raw_venv(name="-venv-with-dash") as venv: - with PipenvInstance(chdir=True, venv_in_project=False, name="-project-with-dash") as p: # venv_root=venv.parent.as_posix(), ignore_virtualenvs=False) as p: + with temp_environ(): + with PipenvInstance(chdir=True, venv_in_project=False, name="-project-with-dash") as p: if "PIPENV_VENV_IN_PROJECT" in os.environ: del os.environ['PIPENV_VENV_IN_PROJECT'] c = p.pipenv('run pip freeze') From 05d3b5af2610a17d4ec09cc8bfb3cede10d93b9e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 24 Feb 2019 22:34:26 -0500 Subject: [PATCH 14/14] Update license download script Signed-off-by: Dan Ryan --- tasks/vendoring/__init__.py | 64 +++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index d6ea9614..dce9e5a7 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -22,6 +22,7 @@ from urllib3.util import parse_url as urllib3_parse from pipenv.utils import mkdir_p from pipenv.vendor.vistir.compat import NamedTemporaryFile, TemporaryDirectory from pipenv.vendor.vistir.contextmanagers import open_file +import pipenv.vendor.parse as parse TASK_NAME = 'update' @@ -458,19 +459,21 @@ def packages_missing_licenses(ctx, vendor_dir=None, requirements_file='vendor.tx possible_pkgs.append(LIBRARY_DIRNAMES[pkg]) for pkgpath in possible_pkgs: pkgpath = vendor_dir.joinpath(pkgpath) + py_path = pkgpath.parent / "{0}.py".format(pkgpath.stem) if pkgpath.exists() and pkgpath.is_dir(): - for licensepath in LICENSES: - licensepath = pkgpath.joinpath(licensepath) - if licensepath.exists(): + for license_path in LICENSES: + license_path = pkgpath.joinpath(license_path) + if license_path.exists(): match_found = True - # log("%s: Trying path %s... FOUND" % (pkg, licensepath)) + # log("%s: Trying path %s... FOUND" % (pkg, license_path)) break - elif (pkgpath.exists() or pkgpath.parent.joinpath("{0}.py".format(pkgpath.stem)).exists()): - for licensepath in LICENSES: - licensepath = pkgpath.parent.joinpath("{0}.{1}".format(pkgpath.stem, licensepath)) - if licensepath.exists(): + elif pkgpath.exists() or py_path.exists(): + for license_path in LICENSES: + license_name = "{0}.{1}".format(pkgpath.stem, license_path) + license_path = pkgpath.parent / license_name + if license_path.exists(): match_found = True - # log("%s: Trying path %s... FOUND" % (pkg, licensepath)) + # log("%s: Trying path %s... FOUND" % (pkg, license_path)) break if match_found: break @@ -483,7 +486,10 @@ def packages_missing_licenses(ctx, vendor_dir=None, requirements_file='vendor.tx @invoke.task -def download_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', package=None, only=False, patched=False): +def download_licenses( + ctx, vendor_dir=None, requirements_file='vendor.txt', package=None, only=False, + patched=False +): log('Downloading licenses') if not vendor_dir: if patched: @@ -509,13 +515,37 @@ def download_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', pack requirement = package tmp_dir = vendor_dir / '__tmp__' # TODO: Fix this whenever it gets sorted out (see https://github.com/pypa/pip/issues/5739) + cmd = "pip download --no-binary :all: --only-binary requests_download --no-deps" + enum_cmd = "pip download --no-deps" ctx.run('pip install flit') # needed for the next step - ctx.run( - 'pip download --no-binary :all: --only-binary requests_download --no-build-isolation --no-deps -d {0} {1}'.format( - tmp_dir.as_posix(), - requirement, - ) - ) + for req in requirements_file.read_text().splitlines(): + if req.startswith("enum34"): + exe_cmd = "{0} -d {1} {2}".format(enum_cmd, tmp_dir.as_posix(), req) + else: + exe_cmd = "{0} --no-build-isolation --no-use-pep517 -d {1} {2}".format( + cmd, tmp_dir.as_posix(), req + ) + try: + ctx.run(exe_cmd) + except invoke.exceptions.UnexpectedExit as e: + if "Disabling PEP 517 processing is invalid" not in e.result.stderr: + log("WARNING: Failed to download license for {0}".format(req)) + continue + parse_target = ( + "Disabling PEP 517 processing is invalid: project specifies a build " + "backend of {backend} in pyproject.toml" + ) + target = parse.parse(parse_target, e.result.stderr.strip()) + backend = target.named.get("backend") + if backend is not None: + if "." in backend: + backend, _, _ = backend.partition(".") + ctx.run("pip install {0}".format(backend)) + ctx.run( + "{0} --no-build-isolation -d {1} {2}".format( + cmd, tmp_dir.as_posix(), req + ) + ) for sdist in tmp_dir.iterdir(): extract_license(vendor_dir, sdist) new_requirements_file.unlink() @@ -527,7 +557,7 @@ def extract_license(vendor_dir, sdist): ext = sdist.suffix[1:] with tarfile.open(sdist, mode='r:{}'.format(ext)) as tar: found = find_and_extract_license(vendor_dir, tar, tar.getmembers()) - elif sdist.suffix == '.zip': + elif sdist.suffix in ('.zip', '.whl'): with zipfile.ZipFile(sdist) as zip: found = find_and_extract_license(vendor_dir, zip, zip.infolist()) else: