From a84f5443af8a3e6615acd1a4afebf96d1af8002d Mon Sep 17 00:00:00 2001 From: Peter Schutt Date: Thu, 15 Nov 2018 10:17:58 +1100 Subject: [PATCH 1/8] Incorrect ref in VCS example Small changes, but a couple of issues in the `A Note About VCS Dependencies` section of the docs. Running the example `$ pipenv install -e git+https://github.com/requests/requests.git@v2.19#egg=requests` raises the error: `Did not find branch or tag 'v2.19', assuming revision or ref.` I think it should be 'v2.19.1' which is just a typo as 'v2.19.1' is used in the subsequent lines of the example. The example Pipfile in the section excludes the `v` in the version number written to the `ref = ` part. I found it a little confusing that it would strip the 'v' from the tag, but then on testing it I found that it doesn't do that: ``` [packages] requests = {editable = true, ref = "v2.20.1", git = "https://github.com/requests/requests.git"} ``` Also, while I was here I figured I'd update the example to the most recent requests release. Thanks. --- docs/basics.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/basics.rst b/docs/basics.rst index ee2e6621..00f4c7d9 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -367,18 +367,18 @@ The only optional section is the ``@`` section. When using git o Note that it is **strongly recommended** that you install any version-controlled dependencies in editable mode, using ``pipenv install -e``, in order to ensure that dependency resolution can be performed with an up to date copy of the repository each time it is performed, and that it includes all known dependencies. -Below is an example usage which installs the git repository located at ``https://github.com/requests/requests.git`` from tag ``v2.19.1`` as package name ``requests``:: +Below is an example usage which installs the git repository located at ``https://github.com/requests/requests.git`` from tag ``v2.20.1`` as package name ``requests``:: - $ pipenv install -e git+https://github.com/requests/requests.git@v2.19#egg=requests + $ pipenv install -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests Creating a Pipfile for this project... - Installing -e git+https://github.com/requests/requests.git@v2.19.1#egg=requests... + Installing -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests... [...snipped...] - Adding -e git+https://github.com/requests/requests.git@v2.19.1#egg=requests to Pipfile's [packages]... + Adding -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests to Pipfile's [packages]... [...] $ cat Pipfile [packages] - requests = {git = "https://github.com/requests/requests.git", editable = true, ref = "2.19.1"} + requests = {git = "https://github.com/requests/requests.git", editable = true, ref = "v2.20.1"} Valid values for ```` include ``git``, ``bzr``, ``svn``, and ``hg``. Valid values for ```` include ``http``, ``https``, ``ssh``, and ``file``. In specific cases you also have access to other schemes: ``svn`` may be combined with ``svn`` as a scheme, and ``bzr`` can be combined with ``sftp`` and ``lp``. From 061869f22e78d3adc286f075976ae27dff52f55d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 00:56:21 -0500 Subject: [PATCH 2/8] Don't disable --user when --system is passed - Fixes #3222 Signed-off-by: Dan Ryan --- news/3222.bugfix.rst | 1 + pipenv/core.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 news/3222.bugfix.rst diff --git a/news/3222.bugfix.rst b/news/3222.bugfix.rst new file mode 100644 index 00000000..2f71f552 --- /dev/null +++ b/news/3222.bugfix.rst @@ -0,0 +1 @@ +Pipenv will no longer disable user-mode installation when the ``--system`` flag is passed in. diff --git a/pipenv/core.py b/pipenv/core.py index f815e5de..8c48d7fa 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -725,7 +725,8 @@ def batch_install(deps_list, procs, failed_deps_queue, extra_indexes = indexes[1:] with vistir.contextmanagers.temp_environ(): - os.environ["PIP_USER"] = vistir.compat.fs_str("0") + if not allow_global: + os.environ["PIP_USER"] = vistir.compat.fs_str("0") c = pip_install( dep, ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), @@ -1899,7 +1900,8 @@ def do_install( ) # pip install: with vistir.contextmanagers.temp_environ(), create_spinner("Installing...") as sp: - os.environ["PIP_USER"] = vistir.compat.fs_str("0") + if not system: + os.environ["PIP_USER"] = vistir.compat.fs_str("0") try: pkg_requirement = Requirement.from_line(pkg_line) except ValueError as e: From e94495ec8da8d0265fa50757aa8a3b3d7cc4e348 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:01:33 -0500 Subject: [PATCH 3/8] Fix circular import Signed-off-by: Dan Ryan --- pipenv/_compat.py | 6 +++--- pipenv/cli/command.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 35ff2188..09f65669 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -14,7 +14,6 @@ import sys import warnings import vistir from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp -from .utils import logging, rmtree try: from tempfile import _infer_return_type @@ -55,6 +54,7 @@ except ImportError: class finalize(object): def __init__(self, *args, **kwargs): + from .utils import logging logging.warn("weakref.finalize unavailable, not cleaning...") def detach(self): @@ -100,7 +100,7 @@ class TemporaryDirectory(object): @classmethod def _cleanup(cls, name, warn_message): - rmtree(name) + vistir.path.rmtree(name) warnings.warn(warn_message, ResourceWarning) def __repr__(self): @@ -114,7 +114,7 @@ class TemporaryDirectory(object): def cleanup(self): if self._finalizer.detach(): - rmtree(self.name) + vistir.path.rmtree(self.name) def _sanitize_params(prefix, suffix, dir): diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 14120e35..5a39fb02 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -15,7 +15,6 @@ import click_completion from click_didyoumean import DYMCommandCollection -from .. import environments from ..__version__ import __version__ from .options import ( CONTEXT_SETTINGS, PipenvGroup, code_option, common_options, deploy_option, @@ -115,6 +114,7 @@ def cli( return 1 if envs: echo("The following environment variables can be set, to do various things:\n") + from .. import environments for key in environments.__dict__: if key.startswith("PIPENV"): echo(" - {0}".format(crayons.normal(key, bold=True))) @@ -161,7 +161,8 @@ def cli( # --rm was passed… elif rm: # Abort if --system (or running in a virtualenv). - if environments.PIPENV_USE_SYSTEM: + from .environments import PIPENV_USE_SYSTEM + if PIPENV_USE_SYSTEM: echo( crayons.red( "You are attempting to remove a virtualenv that " From 6c8af23ec05fb8281f5d7ff0826003e0155b372b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:23:08 -0500 Subject: [PATCH 4/8] Re-try dest distribution Signed-off-by: Dan Ryan --- .azure-pipelines/steps/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index 6515f604..fa45f558 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,5 +1,5 @@ steps: -- powershell: ./DistributeTests.ps1 +- powershell: .\azure-pipelines\steps\DistributeTests.ps1 displayName: 'PowerShell Script to distribute tests' - powershell: | From ef235d8006ca5b898e119b7cbe8681440f9cce5e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:24:49 -0500 Subject: [PATCH 5/8] Fix import Signed-off-by: Dan Ryan --- pipenv/cli/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 5a39fb02..16778ede 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -161,7 +161,7 @@ def cli( # --rm was passed… elif rm: # Abort if --system (or running in a virtualenv). - from .environments import PIPENV_USE_SYSTEM + from ..environments import PIPENV_USE_SYSTEM if PIPENV_USE_SYSTEM: echo( crayons.red( From 5525a87c4b8ad90abfc4646aec38d56817170a42 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:28:03 -0500 Subject: [PATCH 6/8] fix run path for test runner script Signed-off-by: Dan Ryan --- .azure-pipelines/steps/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index fa45f558..cbddf0ee 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,5 +1,5 @@ steps: -- powershell: .\azure-pipelines\steps\DistributeTests.ps1 +- powershell: .\.azure-pipelines\steps\DistributeTests.ps1 displayName: 'PowerShell Script to distribute tests' - powershell: | From 4359376e5a59ea0ef2c40ab8b04d4db91e06bd50 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 15 Nov 2018 01:45:36 -0500 Subject: [PATCH 7/8] dont distribute tests, it doesn't work Signed-off-by: Dan Ryan --- .azure-pipelines/steps/DistributeTests.ps1 | 43 ---------------------- .azure-pipelines/steps/run-tests.yml | 5 +-- 2 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 .azure-pipelines/steps/DistributeTests.ps1 diff --git a/.azure-pipelines/steps/DistributeTests.ps1 b/.azure-pipelines/steps/DistributeTests.ps1 deleted file mode 100644 index d9fce4a3..00000000 --- a/.azure-pipelines/steps/DistributeTests.ps1 +++ /dev/null @@ -1,43 +0,0 @@ -<# -.SYNOPSIS - Distribute the tests in VSTS pipeline across multiple agents -.DESCRIPTION - This script slices tests files across multiple agents for faster execution. - We search for specific type of file structure (in this example test*), and slice them according to agent number - If we encounter multiple files [file1..file10] and if we have 2 agents, agent1 executes tests odd number of files while agent2 executes even number of files - For detalied slicing info: https://docs.microsoft.com/en-us/vsts/pipelines/test/parallel-testing-any-test-runner - We use JUnit style test results to publish the test reports. -#> - -$tests = Get-ChildItem ..\..\tests\unit,..\..\tests\integration -Filter "test*" # search for test files with specific pattern. -$totalAgents = [int]$Env:SYSTEM_TOTALJOBSINPHASE # standard VSTS variables available using parallel execution; total number of parallel jobs running -$agentNumber = [int]$Env:SYSTEM_JOBPOSITIONINPHASE # current job position -$testCount = $tests.Count - -# below conditions are used if parallel pipeline is not used. i.e. pipeline is running with single agent (no parallel configuration) -if ($totalAgents -eq 0) { - $totalAgents = 1 -} -if (!$agentNumber -or $agentNumber -eq 0) { - $agentNumber = 1 -} - -Write-Host "Total agents: $totalAgents" -Write-Host "Agent number: $agentNumber" -Write-Host "Total tests: $testCount" - -$testsToRun= @() - -# slice test files to make sure each agent gets unique test file to execute -For ($i=$agentNumber; $i -le $testCount;) { - $file = $tests[$i-1] - $testsToRun = $testsToRun + $file - Write-Host "Added $file" - $i = $i + $totalAgents - } - -# join all test files seperated by space. pytest runs multiple test files in following format pytest test1.py test2.py test3.py -$testFiles = $testsToRun -Join " " -Write-Host "Test files $testFiles" -# write these files into variable so that we can run them using pytest in subsequent task. -Write-Host "##vso[task.setvariable variable=pytestfiles;]$testFiles" diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index cbddf0ee..7496c76f 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,7 +1,4 @@ steps: -- powershell: .\.azure-pipelines\steps\DistributeTests.ps1 - displayName: 'PowerShell Script to distribute tests' - - powershell: | # Fix Git SSL errors pip install certifi @@ -15,7 +12,7 @@ steps: $env:TEMP='T:\' Write-Host "##vso[task.setvariable variable=TMP]T:\" $env:TMP='T:\' - D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml "$env:pytestfiles" + D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests - task: PublishTestResults@2 From 905c8c062a872a30a1defb5c1ef97c9fd58974da Mon Sep 17 00:00:00 2001 From: frostming Date: Fri, 16 Nov 2018 10:51:18 +0800 Subject: [PATCH 8/8] Derive missing values of source from existing fields Signed-off-by: frostming --- pipenv/project.py | 22 +++++++++++++--------- tests/integration/test_lock.py | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 590fafe5..0067348c 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -727,6 +727,18 @@ class Project(object): data[u"requires"] = {"python_version": version[: len("2.7")]} self.write_toml(data) + @classmethod + def populate_source(cls, source): + """Derive missing values of source from the existing fields.""" + # Only URL pararemter is mandatory, let the KeyError be thrown. + if "name" not in source: + source["name"] = get_url_name(source["url"]) + if "verify_ssl" not in source: + source["verify_ssl"] = "https://" in source["url"] + if not isinstance(source["verify_ssl"], bool): + source["verify_ssl"] = source["verify_ssl"].lower() == "true" + return source + def get_or_create_lockfile(self): from pipenv.vendor.requirementslib.models.lockfile import Lockfile as Req_Lockfile lockfile = None @@ -747,15 +759,7 @@ class Project(object): elif not isinstance(sources, list): sources = [sources,] lockfile_dict["_meta"]["sources"] = [ - { - "name": s.get("name", get_url_name(s.get("url"))), - "url": s["url"], - "verify_ssl": ( - s["verify_ssl"] if isinstance(s["verify_ssl"], bool) else ( - True if s["verify_ssl"].lower() == "true" else False - ) - ) - } for s in sources + self.populate_source(s) for s in sources ] _created_lockfile = Req_Lockfile.from_data( path=self.lockfile_location, data=lockfile_dict, meta_from_project=False diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 2b520808..f2f0ac76 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -486,3 +486,20 @@ def test_lockfile_with_empty_dict(PipenvInstance): assert c.return_code == 0 assert 'Pipfile.lock is corrupted' in c.err assert p.lockfile['_meta'] + + +@pytest.mark.lock +@pytest.mark.install +def test_lock_with_incomplete_source(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[[source]] +url = "https://test.pypi.org/simple" + +[packages] +requests = "*" + """) + c = p.pipenv('install') + assert c.return_code == 0 + assert p.lockfile['_meta']['sources']