From c79b291cbbc0a21119d01912517aae47d0e2de1e Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 3 Apr 2019 09:34:22 +0700 Subject: [PATCH 1/8] Avoid using requests unnecessarily in tests This replaces requests with tablib in test_include_editable_packages as in a de-vendored environment the locking fails with requests. Continuation of https://github.com/pypa/pipenv/issues/3644 --- news/3644.trivial.rst | 1 + tests/integration/test_project.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 news/3644.trivial.rst diff --git a/news/3644.trivial.rst b/news/3644.trivial.rst new file mode 100644 index 00000000..5a7db2e1 --- /dev/null +++ b/news/3644.trivial.rst @@ -0,0 +1 @@ +Use tablib instead of requests in tests to avoid failures when vendored diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index d366ed9d..f7b0e460 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -155,16 +155,16 @@ six = {{version = "*", index = "pypi"}} @pytest.mark.install @pytest.mark.project def test_include_editable_packages(PipenvInstance, testsroot, pathlib_tmpdir): - file_name = "requests-2.19.1.tar.gz" - package = pathlib_tmpdir.joinpath("requests-2.19.1") - source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) + file_name = "tablib-0.12.1.tar.gz" + package = pathlib_tmpdir.joinpath("tablib-0.12.1") + source_path = os.path.abspath(os.path.join(testsroot, "pypi", "tablib", file_name)) with PipenvInstance(chdir=True) as p: with tarfile.open(source_path, "r:gz") as tarinfo: tarinfo.extractall(path=str(pathlib_tmpdir)) c = p.pipenv('install -e {0}'.format(package.as_posix())) assert c.return_code == 0 project = Project() - assert "requests" in [ + assert "tablib" in [ package.project_name for package in project.environment.get_installed_packages() ] From 8d12d8f7f944126a374b58db888902a8a198d1e0 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Thu, 11 Jul 2019 13:31:38 +0700 Subject: [PATCH 2/8] Update tests/pypi to include tablib 0.13.0 --- tests/pypi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pypi b/tests/pypi index 0801b3ae..38f55ba5 160000 --- a/tests/pypi +++ b/tests/pypi @@ -1 +1 @@ -Subproject commit 0801b3aecfbe8385ea879860fb36477a13a4278b +Subproject commit 38f55ba5883f1ce47c6f1f46feecc0d318c444a5 From 050d9b4c784ae0a2312ee636f4147aa396a66184 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 3 Apr 2019 12:31:17 +0700 Subject: [PATCH 3/8] graph tests: Replace requests with tablib 0.13.0 These tests fail when using requests in a de-vendored installation because requests is part of the pre-installed image. tablib has fewer constraints on its dependencies, however its dependency tree is complicated enough to approximately test the same functionality, and allows the graph tests to pass even when de-vendored. --- tests/integration/test_cli.py | 52 ++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index 6d936a7f..1ee9f64a 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -84,46 +84,60 @@ def test_pipenv_rm(PipenvInstance): @pytest.mark.cli def test_pipenv_graph(PipenvInstance): with PipenvInstance() as p: - c = p.pipenv('install requests') + c = p.pipenv('install tablib') assert c.ok graph = p.pipenv("graph") assert graph.ok - assert "requests" in graph.out + assert "tablib" in graph.out graph_json = p.pipenv("graph --json") assert graph_json.ok - assert "requests" in graph_json.out + assert "tablib" in graph_json.out graph_json_tree = p.pipenv("graph --json-tree") assert graph_json_tree.ok - assert "requests" in graph_json_tree.out + assert "tablib" in graph_json_tree.out @pytest.mark.cli def test_pipenv_graph_reverse(PipenvInstance): with PipenvInstance() as p: - c = p.pipenv('install requests==2.18.4') + c = p.pipenv('install tablib==0.13.0') assert c.ok c = p.pipenv('graph --reverse') assert c.ok output = c.out - requests_dependency = [ - ('certifi', 'certifi>=2017.4.17'), - ('chardet', 'chardet(>=3.0.2,<3.1.0|<3.1.0,>=3.0.2)'), - ('idna', 'idna(>=2.5,<2.7|<2.7,>=2.5)'), - ('urllib3', 'urllib3(>=1.21.1,<1.23|<1.23,>=1.21.1)') - ] - - for dep_name, dep_constraint in requests_dependency: - dep_match = re.search(r'^{}==[\d.]+$'.format(dep_name), output, flags=re.MULTILINE) - dep_requests_match = re.search(r'^ - requests==2.18.4 \[requires: {}\]$'.format(dep_constraint), output, flags=re.MULTILINE) - assert dep_match is not None - assert dep_requests_match is not None - assert dep_requests_match.start() > dep_match.start() - c = p.pipenv('graph --reverse --json') assert c.return_code == 1 assert 'Warning: Using both --reverse and --json together is not supported.' in c.err + requests_dependency = [ + ('backports.csv', 'backports.csv'), + ('odfpy', 'odfpy'), + ('openpyxl', 'openpyxl>=2.4.0'), + ('pyyaml', 'pyyaml'), + ('xlrd', 'xlrd'), + ('xlwt', 'xlwt'), + ] + + for dep_name, dep_constraint in requests_dependency: + pat = r'^[ -]*{}==[\d.]+'.format(dep_name) + dep_match = re.search(pat, output, flags=re.MULTILINE) + assert dep_match is not None, '{} not found in {}'.format(pat, output) + + # openpyxl should be indented + if dep_name == 'openpyxl': + openpyxl_dep = re.search(r'^openpyxl', output, flags=re.MULTILINE) + assert openpyxl_dep is None, 'openpyxl should not appear at begining of lines in {}'.format(output) + + assert ' - openpyxl==2.5.4 [requires: et-xmlfile]' in output + else: + dep_match = re.search(r'^[ -]*{}==[\d.]+$'.format(dep_name), output, flags=re.MULTILINE) + assert dep_match is not None, '{} not found at beginning of line in {}'.format(dep_name, output) + + dep_requests_match = re.search(r'^ +- tablib==0.13.0 \[requires: {}\]$'.format(dep_constraint), output, flags=re.MULTILINE) + assert dep_requests_match is not None, 'constraint {} not found in {}'.format(dep_constraint, output) + assert dep_requests_match.start() > dep_match.start() + @pytest.mark.cli @pytest.mark.needs_internet(reason='required by check') From e4d992b84ceb3d5ad01881a03ee6a5f424839fd3 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 3 Apr 2019 18:39:13 +0700 Subject: [PATCH 4/8] test_uninstall.py: Replace requests with Django --- tests/integration/test_uninstall.py | 105 ++++++++++++++++------------ 1 file changed, 61 insertions(+), 44 deletions(-) diff --git a/tests/integration/test_uninstall.py b/tests/integration/test_uninstall.py index ef7fb688..bc82df8d 100644 --- a/tests/integration/test_uninstall.py +++ b/tests/integration/test_uninstall.py @@ -11,30 +11,48 @@ from pipenv.utils import temp_environ @pytest.mark.run @pytest.mark.uninstall @pytest.mark.install -def test_uninstall(PipenvInstance): +def test_uninstall_requests(PipenvInstance): + # Uninstalling requests can fail even when uninstall Django below + # succeeds, if requests was de-vendored. + # See https://github.com/pypa/pipenv/issues/3644 for problems + # caused by devendoring with PipenvInstance() as p: c = p.pipenv("install requests") assert c.return_code == 0 assert "requests" in p.pipfile["packages"] - assert "requests" in p.lockfile["default"] - assert "chardet" in p.lockfile["default"] - assert "idna" in p.lockfile["default"] - assert "urllib3" in p.lockfile["default"] - assert "certifi" in p.lockfile["default"] + + c = p.pipenv("run python -m requests.help") + assert c.return_code == 0 c = p.pipenv("uninstall requests") assert c.return_code == 0 assert "requests" not in p.pipfile["dev-packages"] - assert "requests" not in p.lockfile["develop"] - assert "chardet" not in p.lockfile["develop"] - assert "idna" not in p.lockfile["develop"] - assert "urllib3" not in p.lockfile["develop"] - assert "certifi" not in p.lockfile["develop"] c = p.pipenv("run python -m requests.help") assert c.return_code > 0 +def test_uninstall_django(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv("install Django==1.11.13") + assert c.return_code == 0 + assert "django" in p.pipfile["packages"] + assert "django" in p.lockfile["default"] + assert "pytz" in p.lockfile["default"] + + c = p.pipenv("run python -m django --version") + assert c.return_code == 0 + + c = p.pipenv("uninstall Django") + assert c.return_code == 0 + assert "django" not in p.pipfile["dev-packages"] + assert "django" not in p.lockfile["develop"] + assert p.lockfile["develop"] == {} + + c = p.pipenv("run python -m django --version") + assert c.return_code > 0 + + @pytest.mark.run @pytest.mark.uninstall @pytest.mark.install @@ -46,35 +64,32 @@ def test_mirror_uninstall(PipenvInstance): ) assert "pypi.org" not in mirror_url - c = p.pipenv("install requests --pypi-mirror {0}".format(mirror_url)) + c = p.pipenv("install Django==1.11.13 --pypi-mirror {0}".format(mirror_url)) assert c.return_code == 0 - assert "requests" in p.pipfile["packages"] - assert "requests" in p.lockfile["default"] - assert "chardet" in p.lockfile["default"] - assert "idna" in p.lockfile["default"] - assert "urllib3" in p.lockfile["default"] - assert "certifi" in p.lockfile["default"] + assert "django" in p.pipfile["packages"] + assert "django" in p.lockfile["default"] + assert "pytz" in p.lockfile["default"] # Ensure the --pypi-mirror parameter hasn't altered the Pipfile or Pipfile.lock sources assert len(p.pipfile["source"]) == 1 assert len(p.lockfile["_meta"]["sources"]) == 1 assert "https://pypi.org/simple" == p.pipfile["source"][0]["url"] assert "https://pypi.org/simple" == p.lockfile["_meta"]["sources"][0]["url"] - c = p.pipenv("uninstall requests --pypi-mirror {0}".format(mirror_url)) + c = p.pipenv("run python -m django --version") assert c.return_code == 0 - assert "requests" not in p.pipfile["dev-packages"] - assert "requests" not in p.lockfile["develop"] - assert "chardet" not in p.lockfile["develop"] - assert "idna" not in p.lockfile["develop"] - assert "urllib3" not in p.lockfile["develop"] - assert "certifi" not in p.lockfile["develop"] + + c = p.pipenv("uninstall Django --pypi-mirror {0}".format(mirror_url)) + assert c.return_code == 0 + assert "django" not in p.pipfile["dev-packages"] + assert "django" not in p.lockfile["develop"] + assert p.lockfile["develop"] == {} # Ensure the --pypi-mirror parameter hasn't altered the Pipfile or Pipfile.lock sources assert len(p.pipfile["source"]) == 1 assert len(p.lockfile["_meta"]["sources"]) == 1 assert "https://pypi.org/simple" == p.pipfile["source"][0]["url"] assert "https://pypi.org/simple" == p.lockfile["_meta"]["sources"][0]["url"] - c = p.pipenv("run python -m requests.help") + c = p.pipenv("run python -m django --version") assert c.return_code > 0 @@ -82,21 +97,21 @@ def test_mirror_uninstall(PipenvInstance): @pytest.mark.uninstall @pytest.mark.install def test_uninstall_all_local_files(PipenvInstance, testsroot): - file_name = "requests-2.19.1.tar.gz" + file_name = "tablib-0.12.1.tar.gz" # Not sure where travis/appveyor run tests from - source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) + source_path = os.path.abspath(os.path.join(testsroot, "pypi", "tablib", file_name)) with PipenvInstance(chdir=True) as p: shutil.copy(source_path, os.path.join(p.path, file_name)) - os.mkdir(os.path.join(p.path, "requests")) + os.mkdir(os.path.join(p.path, "tablib")) c = p.pipenv("install {}".format(file_name)) assert c.return_code == 0 c = p.pipenv("uninstall --all") assert c.return_code == 0 - assert "requests" in c.out + assert "tablib" in c.out # Uninstall --all is not supposed to remove things from the pipfile # Note that it didn't before, but that instead local filenames showed as hashes - assert "requests" in p.pipfile["packages"] + assert "tablib" in p.pipfile["packages"] @pytest.mark.run @@ -104,32 +119,34 @@ def test_uninstall_all_local_files(PipenvInstance, testsroot): @pytest.mark.install def test_uninstall_all_dev(PipenvInstance): with PipenvInstance() as p: - c = p.pipenv("install --dev requests six") + c = p.pipenv("install --dev Django==1.11.13 six") assert c.return_code == 0 - c = p.pipenv("install pytz") + c = p.pipenv("install tablib") assert c.return_code == 0 - assert "pytz" in p.pipfile["packages"] - assert "requests" in p.pipfile["dev-packages"] + assert "tablib" in p.pipfile["packages"] + assert "django" in p.pipfile["dev-packages"] assert "six" in p.pipfile["dev-packages"] - assert "pytz" in p.lockfile["default"] - assert "requests" in p.lockfile["develop"] + assert "tablib" in p.lockfile["default"] + assert "django" in p.lockfile["develop"] assert "six" in p.lockfile["develop"] + c = p.pipenv('run python -c "import django"') + assert c.return_code == 0 + c = p.pipenv("uninstall --all-dev") assert c.return_code == 0 - assert "requests" not in p.pipfile["dev-packages"] - assert "six" not in p.pipfile["dev-packages"] - assert "requests" not in p.lockfile["develop"] + assert p.pipfile["dev-packages"] == {} + assert "django" not in p.lockfile["develop"] assert "six" not in p.lockfile["develop"] - assert "pytz" in p.pipfile["packages"] - assert "pytz" in p.lockfile["default"] + assert "tablib" in p.pipfile["packages"] + assert "tablib" in p.lockfile["default"] - c = p.pipenv("run python -m requests.help") + c = p.pipenv('run python -c "import django"') assert c.return_code > 0 - c = p.pipenv('run python -c "import pytz"') + c = p.pipenv('run python -c "import tablib"') assert c.return_code == 0 From 9869fcfffa22d5d2b7f307171002422104715a6c Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 3 Apr 2019 19:45:29 +0700 Subject: [PATCH 5/8] test_update_locks: Replace requests with jdcal Using requests in tests fails when pipenv is de-vendored. In this test, using any package with multiple versions in the local pypi data will suffice to show the functionality does work, but not with requests, or any devendored package. Continuation of https://github.com/pypa/pipenv/issues/3644 --- tests/integration/test_pipenv.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_pipenv.py b/tests/integration/test_pipenv.py index b050a72e..12ae2348 100644 --- a/tests/integration/test_pipenv.py +++ b/tests/integration/test_pipenv.py @@ -62,22 +62,23 @@ requests = "==2.14.0" @pytest.mark.update @pytest.mark.lock def test_update_locks(PipenvInstance): - with PipenvInstance() as p: - c = p.pipenv('install requests==2.14.0') + c = p.pipenv('install jdcal==1.3') assert c.return_code == 0 + assert p.lockfile['default']['jdcal']['version'] == '==1.3' with open(p.pipfile_path, 'r') as fh: pipfile_contents = fh.read() - pipfile_contents = pipfile_contents.replace('==2.14.0', '*') + assert '==1.3' in pipfile_contents + pipfile_contents = pipfile_contents.replace('==1.3', '*') with open(p.pipfile_path, 'w') as fh: fh.write(pipfile_contents) - c = p.pipenv('update requests') + c = p.pipenv('update jdcal') assert c.return_code == 0 - assert p.lockfile['default']['requests']['version'] == '==2.19.1' + assert p.lockfile['default']['jdcal']['version'] == '==1.4' c = p.pipenv('run pip freeze') assert c.return_code == 0 lines = c.out.splitlines() - assert 'requests==2.19.1' in [l.strip() for l in lines] + assert 'jdcal==1.4' in [l.strip() for l in lines] @pytest.mark.project From 2f5150ab29006b7ac2ffa7d5f352ba31c22cb156 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 5 Apr 2019 01:23:28 +0700 Subject: [PATCH 6/8] test_install_without_dev: Replace pytz with tablib pytz exists in the build environment, which will typically be added to the PYTHONPATH when running the tests in rpmbuild. The alternative is to create a virtualenv containing only the pipenv dependencies, which is cumbersome to do when devendoring. As the pytz library here is not critical to the logic of the test method, replace it with tablib to reduce unintentionally errors when packaging. Continuation of https://github.com/pypa/pipenv/issues/3644 --- tests/integration/test_install_basic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index c2709f85..80ccdf0e 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -122,16 +122,16 @@ def test_install_without_dev(PipenvInstance): six = "*" [dev-packages] -pytz = "*" +tablib = "*" """.strip() f.write(contents) c = p.pipenv("install") assert c.return_code == 0 assert "six" in p.pipfile["packages"] - assert "pytz" in p.pipfile["dev-packages"] + assert "tablib" in p.pipfile["dev-packages"] assert "six" in p.lockfile["default"] - assert "pytz" in p.lockfile["develop"] - c = p.pipenv('run python -c "import pytz"') + assert "tablib" in p.lockfile["develop"] + c = p.pipenv('run python -c "import tablib"') assert c.return_code != 0 c = p.pipenv('run python -c "import six"') assert c.return_code == 0 From d503c496b786073225e92ef970c84c8520dbed97 Mon Sep 17 00:00:00 2001 From: Doc Developer <39494509+docdeveloper@users.noreply.github.com> Date: Thu, 11 Jul 2019 21:28:20 +0200 Subject: [PATCH 7/8] Fixes formatting --- docs/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index 8f683722..7ae54df8 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -71,7 +71,7 @@ Luckily - pipenv will hash your Pipfile *before* expanding environment variables (and, helpfully, will substitute the environment variables again when you install from the lock file - so no need to commit any secrets! Woo!) -If your credentials contain a special character, surround the references to the environment variables with quotation marks. For example, if your password contain a double quotation mark, surround the password variable with single quotation marks. Otherwise, you may get a `ValueError, "No closing quotation"` error while installing dependencies. +If your credentials contain a special character, surround the references to the environment variables with quotation marks. For example, if your password contain a double quotation mark, surround the password variable with single quotation marks. Otherwise, you may get a ``ValueError, "No closing quotation"`` error while installing dependencies. :: [[source]] url = "https://$USERNAME:'${PASSWORD}'@mypypi.example.com/simple" From 67452b1f54f80736932721b2c5402b09ceacc480 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 13 Jul 2019 12:11:11 +0800 Subject: [PATCH 8/8] Make sure symlinks are resolved. --- news/3842.bugfix.rst | 1 + pipenv/project.py | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 news/3842.bugfix.rst diff --git a/news/3842.bugfix.rst b/news/3842.bugfix.rst new file mode 100644 index 00000000..fb21be89 --- /dev/null +++ b/news/3842.bugfix.rst @@ -0,0 +1 @@ +Resolve the symlinks when the path is absolute. diff --git a/pipenv/project.py b/pipenv/project.py index d0d668bf..c4b0c941 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -47,11 +47,10 @@ def _normalized(p): if p is None: return None loc = vistir.compat.Path(p) - if not loc.is_absolute(): - try: - loc = loc.resolve() - except OSError: - loc = loc.absolute() + try: + loc = loc.resolve() + except OSError: + loc = loc.absolute() # Recase the path properly on Windows. From https://stackoverflow.com/a/35229734/5043728 if os.name == 'nt': matches = glob.glob(re.sub(r'([^:/\\])(?=[/\\]|$)', r'[\1]', str(loc)))