From 45e9f6e417615ba37f4b0621b9b1194693895c2c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 18 Nov 2018 23:49:57 -0500 Subject: [PATCH 1/4] Fix parsing of markers in VCS requirements - Parsing of markers in non-editable vcs requirements was broken - This PR adds some VCS repos, some utility pipfile generation functions and some fixture helpers - Fixes #3249 Signed-off-by: Dan Ryan --- .gitmodules | 15 +++++++++ news/3249.bugfix.rst | 1 + pipenv/cli/command.py | 1 - pipenv/core.py | 6 +++- pytest.ini | 2 +- tests/conftest.py | 0 tests/integration/conftest.py | 44 +++++++++++++++++++++++++-- tests/integration/test_install_uri.py | 14 +++++++++ tests/test_artifacts/git/dateutil | 1 + tests/test_artifacts/git/pinax | 1 + tests/test_artifacts/git/requests | 1 + tests/test_artifacts/git/six | 1 + tests/test_artifacts/git/six-1.9.0 | 1 + 13 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 .gitmodules create mode 100644 news/3249.bugfix.rst create mode 100644 tests/conftest.py create mode 160000 tests/test_artifacts/git/dateutil create mode 160000 tests/test_artifacts/git/pinax create mode 160000 tests/test_artifacts/git/requests create mode 160000 tests/test_artifacts/git/six create mode 160000 tests/test_artifacts/git/six-1.9.0 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..f40ee10c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,15 @@ +[submodule "tests/test_artifacts/git/six-1.9.0"] + path = tests/test_artifacts/git/six-1.9.0 + url = https://github.com/benjaminp/six.git +[submodule "tests/test_artifacts/git/pinax"] + path = tests/test_artifacts/git/pinax + url = https://github.com/pinax/pinax.git +[submodule "tests/test_artifacts/git/requests"] + path = tests/test_artifacts/git/requests + url = https://github.com/requests/requests.git +[submodule "tests/test_artifacts/git/six"] + path = tests/test_artifacts/git/six + url = https://github.com/benjaminp/six.git +[submodule "tests/test_artifacts/git/dateutil"] + path = tests/test_artifacts/git/dateutil + url = https://github.com/dateutil/dateutil diff --git a/news/3249.bugfix.rst b/news/3249.bugfix.rst new file mode 100644 index 00000000..26d708cb --- /dev/null +++ b/news/3249.bugfix.rst @@ -0,0 +1 @@ +Adding normal pep 508 compatible markers is now fully functional when using VCS dependencies. diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 16778ede..a49e90dd 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -296,7 +296,6 @@ def uninstall( if retcode: sys.exit(retcode) - @cli.command(short_help="Generates Pipfile.lock.") @lock_options @pass_state diff --git a/pipenv/core.py b/pipenv/core.py index 3b304625..7dbd7555 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1361,7 +1361,11 @@ def pip_install( ignore_hashes = True else: ignore_hashes = True if not requirement.hashes else False - install_reqs = [escape_cmd(r) for r in requirement.as_line(as_list=True)] + install_reqs = requirement.as_line(as_list=True) + if not requirement.markers: + install_reqs = [escape_cmd(r) for r in install_reqs] + elif len(install_reqs) > 1: + install_reqs = install_reqs[0] + [escape_cmd(r) for r in install_reqs[1:]] pip_command = [which_pip(allow_global=allow_global), "install"] if pre: pip_command.append("--pre") diff --git a/pytest.ini b/pytest.ini index 2bfca079..ff9847d3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,7 @@ addopts = -ra -n auto testpaths = tests ; Add vendor and patched in addition to the default list of ignored dirs -norecursedirs = .* build dist CVS _darcs {arch} *.egg vendor patched news tasks docs +norecursedirs = .* build dist CVS _darcs {arch} *.egg vendor patched news tasks docs tests/test_artifacts filterwarnings = ignore::DeprecationWarning ignore::PendingDeprecationWarning diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6ed95b3e..33fc4fca 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -9,6 +9,7 @@ from pipenv._compat import TemporaryDirectory, Path from pipenv.vendor import delegator from pipenv.vendor import requests from pipenv.vendor import toml +from pipenv.vendor import tomlkit from pytest_pypi.app import prepare_packages as prepare_pypi_packages from vistir.compat import ResourceWarning, fs_str from vistir.path import mkdir_p @@ -119,6 +120,44 @@ WE_HAVE_INTERNET = check_internet() WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() +class _Pipfile(object): + def __init__(self, path): + self.path = path + self.document = tomlkit.document() + self.document["sources"] = tomlkit.aot() + self.document["requires"] = tomlkit.table() + self.document["packages"] = tomlkit.table() + self.document["dev_packages"] = tomlkit.table() + + def install(self, package, value, dev=False): + section = "packages" if not dev else "dev_packages" + if isinstance(value, dict): + table = tomlkit.inline_table() + table.update(value) + self.document[section][package] = table + else: + self.document[section][package] = value + self.write() + + def loads(self): + self.document = tomlkit.loads(self.path.read_text()) + + def dumps(self): + source_table = tomlkit.table() + source_table["url"] = os.environ.get("PIPENV_TEST_INDEX") + source_table["verify_ssl"] = False + source_table["name"] = "pipenv_test_index" + self.document["sources"].append(source_table) + return tomlkit.dumps(self.document) + + def write(self): + self.path.write_text(self.dumps()) + + @classmethod + def get_fixture_path(cls, path): + return Path(__file__).absolute().parent.parent / "test_artifacts" / path + + class _PipenvInstance(object): """An instance of a Pipenv Project...""" def __init__(self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None): @@ -129,7 +168,7 @@ class _PipenvInstance(object): os.environ["CI"] = fs_str("1") warnings.simplefilter("ignore", category=ResourceWarning) warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") - path = os.environ.get("PIPENV_PROJECT_DIR", None) + path = path if path else os.environ.get("PIPENV_PROJECT_DIR", None) if not path: self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') path = Path(self._path.name) @@ -138,7 +177,7 @@ class _PipenvInstance(object): except OSError: self.path = str(path.absolute()) else: - self._path = None + self._path = path self.path = path # set file creation perms self.pipfile_path = None @@ -154,6 +193,7 @@ class _PipenvInstance(object): self.chdir = False or chdir self.pipfile_path = p_path + self._pipfile = _Pipfile(Path(p_path)) def __enter__(self): os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index c5915607..337181fa 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -270,3 +270,17 @@ PyInstaller = {ref = "develop", git = "https://github.com/pyinstaller/pyinstalle p.lockfile["default"]["pyinstaller"]["git"] == "https://github.com/pyinstaller/pyinstaller.git" ) + + +@pytest.mark.vcs +@pytest.mark.install +@pytest.mark.needs_internet +def test_vcs_can_use_markers(PipenvInstance, pip_src_dir, pypi): + with PipenvInstance(chdir=True, pypi=pypi) as p: + path = p._pipfile.get_fixture_path("git/six/.git") + p._pipfile.install("six", {"git": "{0}".format(path.as_uri()), "markers": "sys_platform == 'linux'"}) + assert "six" in p.pipfile["packages"] + c = p.pipenv("install") + assert c.return_code == 0 + assert "six" in p.lockfile["default"] + assert "git" in p.lockfile["default"]["six"] diff --git a/tests/test_artifacts/git/dateutil b/tests/test_artifacts/git/dateutil new file mode 160000 index 00000000..6618dee9 --- /dev/null +++ b/tests/test_artifacts/git/dateutil @@ -0,0 +1 @@ +Subproject commit 6618dee970ec1e5f92e0f48ec74584caf13075aa diff --git a/tests/test_artifacts/git/pinax b/tests/test_artifacts/git/pinax new file mode 160000 index 00000000..147d8543 --- /dev/null +++ b/tests/test_artifacts/git/pinax @@ -0,0 +1 @@ +Subproject commit 147d854322cc2ab8246b9f52fa3189dfca3cddec diff --git a/tests/test_artifacts/git/requests b/tests/test_artifacts/git/requests new file mode 160000 index 00000000..57d7284c --- /dev/null +++ b/tests/test_artifacts/git/requests @@ -0,0 +1 @@ +Subproject commit 57d7284c1a245cf9fbcecb594f50471d86e879f7 diff --git a/tests/test_artifacts/git/six b/tests/test_artifacts/git/six new file mode 160000 index 00000000..e114efce --- /dev/null +++ b/tests/test_artifacts/git/six @@ -0,0 +1 @@ +Subproject commit e114efceea962fb143c909c904157ca994246fd2 diff --git a/tests/test_artifacts/git/six-1.9.0 b/tests/test_artifacts/git/six-1.9.0 new file mode 160000 index 00000000..5efb522b --- /dev/null +++ b/tests/test_artifacts/git/six-1.9.0 @@ -0,0 +1 @@ +Subproject commit 5efb522b0647f7467248273ec1b893d06b984a59 From d407ac4740b002048606526b683acb534a759f99 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 18 Nov 2018 23:55:50 -0500 Subject: [PATCH 2/4] Add submodule sync and update to scripts Signed-off-by: Dan Ryan --- .azure-pipelines/jobs/run-tests.yml | 1 + .azure-pipelines/steps/run-tests.yml | 2 + run-tests.bat | 14 +++---- run-tests.sh | 3 ++ run_local_tests.sh | 57 ++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 7 deletions(-) create mode 100755 run_local_tests.sh diff --git a/.azure-pipelines/jobs/run-tests.yml b/.azure-pipelines/jobs/run-tests.yml index fe98e8ec..3544af41 100644 --- a/.azure-pipelines/jobs/run-tests.yml +++ b/.azure-pipelines/jobs/run-tests.yml @@ -26,6 +26,7 @@ steps: export GIT_SSL_CAINFO="$(python -m certifi)" export LANG="C.UTF-8" export PIP_PROCESS_DEPENDENCY_LINKS="1" + git submodule sync && git submodule update --init --recursive pipenv run pytest --junitxml=test-results.xml displayName: Run integration tests diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index 7496c76f..f33523f1 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -12,6 +12,8 @@ steps: $env:TEMP='T:\' Write-Host "##vso[task.setvariable variable=TMP]T:\" $env:TMP='T:\' + git submodule sync + git submodule update --init --recursive D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests diff --git a/run-tests.bat b/run-tests.bat index f31a562a..d20af35f 100644 --- a/run-tests.bat +++ b/run-tests.bat @@ -1,7 +1,7 @@ -rem imdisk -a -s 964515b -m R: -p "/FS:NTFS /Y" - -virtualenv R:\.venv -R:\.venv\Scripts\pip install -e . --upgrade --upgrade-strategy=only-if-needed -R:\.venv\Scripts\pipenv install --dev - -SET RAM_DISK=R: && R:\.venv\Scripts\pipenv run pytest -n auto -v tests --tap-stream > report.tap +rem imdisk -a -s 964515b -m R: -p "/FS:NTFS /Y" + +virtualenv R:\.venv +R:\.venv\Scripts\pip install -e . --upgrade --upgrade-strategy=only-if-needed +R:\.venv\Scripts\pipenv install --dev +git submodule sync && git submodule update --init --recursive +SET RAM_DISK=R: && R:\.venv\Scripts\pipenv run pytest -n auto -v tests --tap-stream > report.tap diff --git a/run-tests.sh b/run-tests.sh index 16755623..991c0df7 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -55,6 +55,9 @@ PIPENV_PYTHON=2.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --de PIPENV_PYTHON=3.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --dev PIPENV_PYTHON=2.7 python3 -m pipenv run pip install --upgrade -e . PIPENV_PYTHON=3.7 python3 -m pipenv run pip install --upgrade -e . +echo "$ git submodule sync && git submodule update --init --recursive" + +git submodule sync && git submodule update --init --recursive echo "$ pipenv run time pytest -v -n auto tests -m \"$TEST_SUITE\"" # PIPENV_PYTHON=2.7 pipenv run time pytest -v -n auto tests -m "$TEST_SUITE" | prefix 2.7 & diff --git a/run_local_tests.sh b/run_local_tests.sh new file mode 100755 index 00000000..d3ecb3d6 --- /dev/null +++ b/run_local_tests.sh @@ -0,0 +1,57 @@ +#!/bin/bash +owner=$USER +IFS=' ' read -a groups <<< $(groups) +group=${groups[0]} +python=$(python -c "import sys; print(sys.executable)") +python2=$(python2.7 -c "import sys; print(sys.executable)") +#python3=$(pyenv which python3.7) +ramdisk="/mnt/ramdisk" +venv_path="$ramdisk/.venv" +venv_bin="$venv_path/bin" +# pip="$venv_bin/pip" +# pipenv="$venv_bin/pipenv" +py="$venv_bin/python" +pip="$py -m pip" +pipenv="$py -m pipenv" +venv2_path="$ramdisk/.venv2" +# pipenv2="$venv2_bin/pipenv" +# pip2="$venv2_bin/pip" +venv2_bin="$venv2_path/bin" +py2="$venv2_bin/python" +pip2="$py2 -m pip" +pipenv2="$py2 -m pipenv" +#venv3_path="$ramdisk/.venv3" +#venv3_bin="$venv3_path/bin" +#pip3="$venv3_bin/pip" +#pipenv3="$venv3_bin/pipenv" + + +export RAM_DISK="/mnt/ramdisk/" + +[ ! -e $venv_path ] && sudo mount -t ramfs -o size=2g ramfs $ramdisk \ + && sudo chown -R $owner:$group $ramdisk + +[ ! -e $venv_path ] && python -m virtualenv --python=$python $venv_path \ + && VIRTUAL_ENV="$venv_path" $pip install -e . \ + && VIRTUAL_ENV="$venv_path" $pipenv run pip install -e . \ + && VIRTUAL_ENV="$venv_path" $pipenv install --dev + +[ ! -e $venv2_path ] && python -m virtualenv --python=$python2 $venv2_path \ + && VIRTUAL_ENV="$venv2_path" $pip2 install pathlib2 -e . \ + && VIRTUAL_ENV="$venv2_path" $pipenv2 run pip install -e . \ + && VIRTUAL_ENV="$venv2_path" $pipenv2 install --dev + +#[ ! -e $venv3_path ] && python -m virtualenv --python=$python3 $venv3_path \ + #&& $pip3 install -e . \ + #&& $pipenv3 run pip install -e . \ + #&& $pipenv3 install --dev + + +#export PYPI_VENDOR_DIR="$(pwd)/tests/pypi/" +PIP_PROCESS_DEPENDENCY_LINKS=1 && VIRTUAL_ENV="$venv2_path" $pipenv run pytest tests/ $@ 2>&1 +PIP_PROCESS_DEPENDENCY_LINKS=1 && VIRTUAL_ENV="$venv2_path" $pipenv2 run pytest tests/ $@ 2>&1 +#$pipenv run pytest -v -n 4 --ignore=pipenv/vendor --ignore=pipenv/patched --ignore=build --ignore=tests/pypi --ignore=tests/pytest-pypi/pypi -p tests.pytest-pypi.pytest_pypi tests/ $@ 2>&1 +#$pipenv2 run pytest -v -n 4 --ignore=pipenv/vendor --ignore=pipenv/patched --ignore=build --ignore=tests/pypi --ignore=tests/pytest-pypi/pypi -p tests.pytest-pypi.pytest_pypi tests/ $@ 2>&1 +#$pipenv3 run pytest -v -n 4 --ignore=pipenv/vendor --ignore=pipenv/patched tests/ $@ 2>&1 +# && /mnt/ramdisk/.venv/bin/pip install dist/pipenv-11.10.1.dev1-py2.py3-none-any.whl \ +# && /mnt/ramdisk/.venv/bin/pipenv run pip install dist/pipenv-11.10.1.dev1-py2.py3-none-any.whl \ From 200775e05ce763ab0707f7611ddceeda10899fea Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 19 Nov 2018 01:15:06 -0500 Subject: [PATCH 3/4] Add manifest Signed-off-by: Dan Ryan --- MANIFEST.in | 1 + run_local_tests.sh | 57 ---------------------------------------------- 2 files changed, 1 insertion(+), 57 deletions(-) delete mode 100755 run_local_tests.sh diff --git a/MANIFEST.in b/MANIFEST.in index 3c8eb1d4..30d340ec 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,6 +15,7 @@ include pipenv/vendor/pipreqs/stdlib pipenv/vendor/pipreqs/mapping include pipenv/vendor/*.txt pipenv/vendor/pexpect/bashrc.sh include pipenv/vendor/Makefile include pipenv/pipenv.1 +inlcude .gitmodules exclude .editorconfig .travis.yml .env appveyor.yml tox.ini pytest.ini exclude Pipfile* CHANGELOG.draft.rst exclude docker-compose.yml Dockerfile diff --git a/run_local_tests.sh b/run_local_tests.sh deleted file mode 100755 index d3ecb3d6..00000000 --- a/run_local_tests.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -owner=$USER -IFS=' ' read -a groups <<< $(groups) -group=${groups[0]} -python=$(python -c "import sys; print(sys.executable)") -python2=$(python2.7 -c "import sys; print(sys.executable)") -#python3=$(pyenv which python3.7) -ramdisk="/mnt/ramdisk" -venv_path="$ramdisk/.venv" -venv_bin="$venv_path/bin" -# pip="$venv_bin/pip" -# pipenv="$venv_bin/pipenv" -py="$venv_bin/python" -pip="$py -m pip" -pipenv="$py -m pipenv" -venv2_path="$ramdisk/.venv2" -# pipenv2="$venv2_bin/pipenv" -# pip2="$venv2_bin/pip" -venv2_bin="$venv2_path/bin" -py2="$venv2_bin/python" -pip2="$py2 -m pip" -pipenv2="$py2 -m pipenv" -#venv3_path="$ramdisk/.venv3" -#venv3_bin="$venv3_path/bin" -#pip3="$venv3_bin/pip" -#pipenv3="$venv3_bin/pipenv" - - -export RAM_DISK="/mnt/ramdisk/" - -[ ! -e $venv_path ] && sudo mount -t ramfs -o size=2g ramfs $ramdisk \ - && sudo chown -R $owner:$group $ramdisk - -[ ! -e $venv_path ] && python -m virtualenv --python=$python $venv_path \ - && VIRTUAL_ENV="$venv_path" $pip install -e . \ - && VIRTUAL_ENV="$venv_path" $pipenv run pip install -e . \ - && VIRTUAL_ENV="$venv_path" $pipenv install --dev - -[ ! -e $venv2_path ] && python -m virtualenv --python=$python2 $venv2_path \ - && VIRTUAL_ENV="$venv2_path" $pip2 install pathlib2 -e . \ - && VIRTUAL_ENV="$venv2_path" $pipenv2 run pip install -e . \ - && VIRTUAL_ENV="$venv2_path" $pipenv2 install --dev - -#[ ! -e $venv3_path ] && python -m virtualenv --python=$python3 $venv3_path \ - #&& $pip3 install -e . \ - #&& $pipenv3 run pip install -e . \ - #&& $pipenv3 install --dev - - -#export PYPI_VENDOR_DIR="$(pwd)/tests/pypi/" -PIP_PROCESS_DEPENDENCY_LINKS=1 && VIRTUAL_ENV="$venv2_path" $pipenv run pytest tests/ $@ 2>&1 -PIP_PROCESS_DEPENDENCY_LINKS=1 && VIRTUAL_ENV="$venv2_path" $pipenv2 run pytest tests/ $@ 2>&1 -#$pipenv run pytest -v -n 4 --ignore=pipenv/vendor --ignore=pipenv/patched --ignore=build --ignore=tests/pypi --ignore=tests/pytest-pypi/pypi -p tests.pytest-pypi.pytest_pypi tests/ $@ 2>&1 -#$pipenv2 run pytest -v -n 4 --ignore=pipenv/vendor --ignore=pipenv/patched --ignore=build --ignore=tests/pypi --ignore=tests/pytest-pypi/pypi -p tests.pytest-pypi.pytest_pypi tests/ $@ 2>&1 -#$pipenv3 run pytest -v -n 4 --ignore=pipenv/vendor --ignore=pipenv/patched tests/ $@ 2>&1 -# && /mnt/ramdisk/.venv/bin/pip install dist/pipenv-11.10.1.dev1-py2.py3-none-any.whl \ -# && /mnt/ramdisk/.venv/bin/pipenv run pip install dist/pipenv-11.10.1.dev1-py2.py3-none-any.whl \ From de22ff69db65050f15b645d3d506b38ea4b6ab8f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 19 Nov 2018 09:19:32 -0500 Subject: [PATCH 4/4] Exclude gitmodules Signed-off-by: Dan Ryan --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 30d340ec..5e012535 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,7 +15,7 @@ include pipenv/vendor/pipreqs/stdlib pipenv/vendor/pipreqs/mapping include pipenv/vendor/*.txt pipenv/vendor/pexpect/bashrc.sh include pipenv/vendor/Makefile include pipenv/pipenv.1 -inlcude .gitmodules +exclude .gitmodules exclude .editorconfig .travis.yml .env appveyor.yml tox.ini pytest.ini exclude Pipfile* CHANGELOG.draft.rst exclude docker-compose.yml Dockerfile