From 0164423ac6a498686e75d9b5add895c8bb6f4ca8 Mon Sep 17 00:00:00 2001 From: Charles Lai Date: Sun, 8 Jul 2018 23:43:35 -0700 Subject: [PATCH 01/27] Add docstrings to show short_help when running --help --- pipenv/cli.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pipenv/cli.py b/pipenv/cli.py index fb0b2b2d..1b774508 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -394,6 +394,7 @@ def install( keep_outdated=False, selective_upgrade=False, ): + """Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.""" from .core import do_install do_install( @@ -482,6 +483,7 @@ def uninstall( keep_outdated=False, pypi_mirror=None, ): + """Un-installs a provided package and removes it from Pipfile.""" from .core import do_uninstall do_uninstall( @@ -561,6 +563,7 @@ def lock( pre=False, keep_outdated=False, ): + """Generates Pipfile.lock.""" from .core import ensure_project, do_init, do_lock # Ensure that virtualenv is available. @@ -621,6 +624,7 @@ def shell( anyway=False, pypi_mirror=None, ): + """Spawns a shell within the virtualenv.""" from .core import load_dot_env, do_shell # Prevent user from activating nested environments. @@ -683,6 +687,7 @@ def shell( help="Specify a PyPI mirror.", ) def run(command, args, three=None, python=False, pypi_mirror=None): + """Spawns a command installed into the virtualenv.""" from .core import do_run do_run( @@ -738,6 +743,7 @@ def check( args=None, pypi_mirror=None, ): + """Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.""" from .core import do_check do_check( @@ -827,6 +833,7 @@ def update( outdated=False, more_packages=None, ): + """Runs lock, then sync.""" from .core import ( ensure_project, do_outdated, @@ -891,6 +898,7 @@ def update( @option("--json-tree", is_flag=True, default=False, help="Output JSON in nested tree.") @option("--reverse", is_flag=True, default=False, help="Reversed dependency graph.") def graph(bare=False, json=False, json_tree=False, reverse=False): + """Displays currently-installed dependency graph information.""" from .core import do_graph do_graph(bare=bare, json=json, json_tree=json_tree, reverse=reverse) @@ -919,6 +927,7 @@ def graph(bare=False, json=False, json_tree=False, reverse=False): ) @argument("module", nargs=1) def run_open(module, three=None, python=None, pypi_mirror=None): + """View a given module in your editor.""" from .core import which, ensure_project # Ensure that virtualenv is available. @@ -1000,6 +1009,7 @@ def sync( sequential=False, pypi_mirror=None, ): + """Installs all packages specified in Pipfile.lock.""" from .core import do_sync do_sync( @@ -1045,6 +1055,7 @@ def sync( def clean( ctx, three=None, python=None, dry_run=False, bare=False, user=False, verbose=False ): + """Uninstalls all packages not specified in Pipfile.lock.""" from .core import do_clean do_clean(ctx=ctx, three=three, python=python, dry_run=dry_run, verbose=verbose) From f0a1413c70847c6604f3b9a0a1944f9afa07b704 Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Tue, 10 Jul 2018 20:41:34 -0500 Subject: [PATCH 02/27] [#2504] Add test for VCS dep with extras --- tests/integration/test_lock.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 1ab34f1b..eebfaf39 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -1,6 +1,5 @@ import pytest import os -import six from pipenv.utils import temp_environ @@ -348,6 +347,28 @@ requests = {git = "https://github.com/requests/requests.git", ref = "master", ed assert c.return_code == 0 +@pytest.mark.extras +@pytest.mark.lock +@pytest.mark.vcs +@pytest.mark.needs_internet +def test_lock_editable_vcs_with_extras_without_install(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[packages] +requests = {git = "https://github.com/requests/requests.git", editable = true, extras = ["security"]} + """.strip()) + c = p.pipenv('lock') + assert c.return_code == 0 + assert 'requests' in p.lockfile['default'] + assert 'idna' in p.lockfile['default'] + assert 'chardet' in p.lockfile['default'] + assert 'cryptography' in p.lockfile['default'] + assert 'pyOpenSSL' in p.lockfile['default'] + c = p.pipenv('install') + assert c.return_code == 0 + + @pytest.mark.lock @pytest.mark.skip(reason="This doesn't work for some reason.") def test_lock_respecting_python_version(PipenvInstance, pypi): From 6e05a3e0d27182b21381455751fcff0ec57f7d53 Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Tue, 10 Jul 2018 20:58:57 -0500 Subject: [PATCH 03/27] [#2504] Serialize prettytoml ArrayElements as lists --- pipenv/project.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pipenv/project.py b/pipenv/project.py index 7f40576d..d0d21c3e 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -14,6 +14,7 @@ import pipfile.api import six import toml import json as simplejson +from prettytoml.elements.array import ArrayElement from ._compat import Path @@ -64,6 +65,12 @@ def _normalized(p): DEFAULT_NEWLINES = u"\n" +def encode_toml_elements(obj): + if isinstance(obj, ArrayElement): + return obj.primitive_value + raise TypeError(repr(obj) + " is not JSON serializable") + + def preferred_newlines(f): if isinstance(f.newlines, six.text_type): return f.newlines @@ -631,7 +638,8 @@ class Project(object): """ newlines = self._lockfile_newlines s = simplejson.dumps( # Send Unicode in to guarentee Unicode out. - content, indent=4, separators=(u",", u": "), sort_keys=True + content, indent=4, separators=(u",", u": "), sort_keys=True, + default=encode_toml_elements, ) with atomic_open_for_write(self.lockfile_location, newline=newlines) as f: f.write(s) From 6de915ab79f5e23383f510ef70b6b389a40e51e8 Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Wed, 11 Jul 2018 08:19:42 -0500 Subject: [PATCH 04/27] [#2504] Replace security extra with socks in test --- tests/integration/test_lock.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index eebfaf39..67da46b9 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -356,15 +356,14 @@ def test_lock_editable_vcs_with_extras_without_install(PipenvInstance, pypi): with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git", editable = true, extras = ["security"]} +requests = {git = "https://github.com/requests/requests.git", editable = true, extras = ["socks"]} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 assert 'requests' in p.lockfile['default'] assert 'idna' in p.lockfile['default'] assert 'chardet' in p.lockfile['default'] - assert 'cryptography' in p.lockfile['default'] - assert 'pyOpenSSL' in p.lockfile['default'] + assert 'pysocks' in p.lockfile['default'] c = p.pipenv('install') assert c.return_code == 0 From f7ee0764e3161211df5698d391e315ee1d80a259 Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Wed, 11 Jul 2018 13:24:41 -0500 Subject: [PATCH 05/27] Add pytest default norecursedirs --- pytest.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 92e72fd5..80c3e3a3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,4 @@ [pytest] addopts = -n auto -norecursedirs = vendor patched +; Add vendor and patched in addition to the default list of ignored dirs +norecursedirs = .* build dist CVS _darcs {arch} *.egg vendor patched From dd4b4be175757cd9de45650b9f9bc493cdf7f221 Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Wed, 11 Jul 2018 13:25:02 -0500 Subject: [PATCH 06/27] Remove explicit sets of PYPI_VENDOR_DIR and let integration conftest do its thing --- .env | 1 - run-tests.bat | 2 +- run-tests.sh | 3 --- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.env b/.env index 189ddda7..d77e8f12 100644 --- a/.env +++ b/.env @@ -1,2 +1 @@ HELLO=WORLD -PYPI_VENDOR_DIR="./tests/pypi/" \ No newline at end of file diff --git a/run-tests.bat b/run-tests.bat index 4ddcee2b..f31a562a 100644 --- a/run-tests.bat +++ b/run-tests.bat @@ -4,4 +4,4 @@ virtualenv R:\.venv R:\.venv\Scripts\pip install -e . --upgrade --upgrade-strategy=only-if-needed R:\.venv\Scripts\pipenv install --dev -SET RAM_DISK=R:&& SET PYPI_VENDOR_DIR=".\tests\pypi\" && R:\.venv\Scripts\pipenv run pytest -n auto -v tests --tap-stream > report.tap +SET RAM_DISK=R: && R:\.venv\Scripts\pipenv run pytest -n auto -v tests --tap-stream > report.tap diff --git a/run-tests.sh b/run-tests.sh index 493a90a8..b71fd4fb 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -4,9 +4,6 @@ set -eo pipefail -# Set the PYPI vendor URL for pytest-pypi. -PYPI_VENDOR_DIR="$(pwd)/tests/pypi/" -export PYPI_VENDOR_DIR export PYTHONIOENCODING="utf-8" export LANG=C.UTF-8 From 3a25846f8805c556cf8a6e8f67fd6def0a57560d Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Wed, 11 Jul 2018 13:26:03 -0500 Subject: [PATCH 07/27] Add dev setup and testing section to CONTRIBUTING.md --- CONTRIBUTING.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7fa45c6..bb17336e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,3 +55,45 @@ Please be aware of the following things when filing bug reports: If you do not provide all of these things, it will take us much longer to fix your problem. If we ask you to clarify these and you never respond, we will close your issue without fixing it. + +## Development Setup + +To get your development environment setup, run: + +```sh +pip install -e . +pipenv install --dev +``` + +This will install the repo version of Pipenv and then install the development +dependencies. Once that has completed, you can start developing. + +The repo version of Pipenv must be installed over other global versions to +resolve conflicts with the `pipenv` folder being implicitly added to `sys.path`. +See pypa/pipenv#2557 for more details. + +### Testing + +Tests are written in `pytest` style and can be run very simply: + +```sh +pytest +``` + +This will run all Pipenv tests, which can take awhile. To run a subset of the +tests, the standard pytest filters are available, such as: + +- provide a directory or file: `pytest tests/unit` or `pytest tests/unit/test_cmdparse.py` +- provide a keyword expression: `pytest -k test_lock_editable_vcs_without_install` +- provide a nodeid: `pytest tests/unit/test_cmdparse.py::test_parse` +- provide a test marker: `pytest -m lock` + +#### Package Index + +To speed up testing, tests that rely on a package index for locking and +installing use a local server that contains vendored packages in the +`tests/pypi` directory. Each vendored package should have it's own folder +containing the necessary releases. When adding a release for a package, it is +easiest to use either the `.tar.gz` or universal wheels (ex: `py2.py3-none`). If +a `.tar.gz` or universal wheel is not available, add wheels for all available +architectures and platforms. From aaac6c64292f4457dedf7abdf21abd3cd4597e3a Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Wed, 11 Jul 2018 15:16:46 -0500 Subject: [PATCH 08/27] Replace test PYPI_VENDOR_DIR envvar with explicit package prep --- tests/integration/conftest.py | 4 +- tests/pytest-pypi/pytest_pypi/app.py | 57 +++++++++++-------- .../pytest_pypi/templates/package.html | 4 +- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d71f0f89..8a8aebe6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -9,6 +9,7 @@ from pipenv.vendor import delegator from pipenv.vendor import requests from pipenv.vendor import six from pipenv.vendor import toml +from pytest_pypi.app import prepare_packages as prepare_pypi_packages if six.PY2: class ResourceWarning(Warning): @@ -30,6 +31,8 @@ def check_internet(): WE_HAVE_INTERNET = check_internet() TESTS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PYPI_VENDOR_DIR = os.path.join(TESTS_ROOT, 'pypi') +prepare_pypi_packages(PYPI_VENDOR_DIR) def pytest_runtest_setup(item): @@ -68,7 +71,6 @@ class _PipenvInstance(object): os.environ['PIPENV_DONT_USE_PYENV'] = '1' os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' os.environ['PIPENV_VENV_IN_PROJECT'] = '1' - os.environ['PYPI_VENDOR_DIR'] = os.path.join(TESTS_ROOT, 'pypi') if self.chdir: os.chdir(self.path) return self diff --git a/tests/pytest-pypi/pytest_pypi/app.py b/tests/pytest-pypi/pytest_pypi/app.py index 57a368b8..7ac1bc4b 100644 --- a/tests/pytest-pypi/pytest_pypi/app.py +++ b/tests/pytest-pypi/pytest_pypi/app.py @@ -4,9 +4,6 @@ import json import requests from flask import Flask, redirect, abort, render_template, send_file, jsonify -PYPI_VENDOR_DIR = os.environ.get('PYPI_VENDOR_DIR', './pypi') -PYPI_VENDOR_DIR = os.path.abspath(PYPI_VENDOR_DIR) - app = Flask(__name__) session = requests.Session() @@ -14,41 +11,49 @@ packages = {} class Package(object): - """docstring for Package""" + """Package represents a collection of releases from one or more directories""" def __init__(self, name): super(Package, self).__init__() self.name = name - self._releases = [] + self.releases = {} + self._package_dirs = set() @property - def releases(self): - r = [] - for release in self._releases: - release = release[len(PYPI_VENDOR_DIR):].replace('\\', '/') - r.append(release) - return r + def json(self): + for path in self._package_dirs: + try: + with open(os.path.join(path, 'api.json')) as f: + return json.load(f) + except FileNotFoundError: + pass def __repr__(self): return "/json') def json_for_package(package): try: - with open(os.path.sep.join([PYPI_VENDOR_DIR, package, 'api.json'])) as f: - return jsonify(json.load(f)) + return jsonify(packages[package].json) except Exception: pass r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) return jsonify(r.json()) + if __name__ == '__main__': + PYPI_VENDOR_DIR = os.environ.get('PYPI_VENDOR_DIR', './pypi') + PYPI_VENDOR_DIR = os.path.abspath(PYPI_VENDOR_DIR) + prepare_packages(PYPI_VENDOR_DIR) + app.run() diff --git a/tests/pytest-pypi/pytest_pypi/templates/package.html b/tests/pytest-pypi/pytest_pypi/templates/package.html index 36e70a88..26ba9eca 100644 --- a/tests/pytest-pypi/pytest_pypi/templates/package.html +++ b/tests/pytest-pypi/pytest_pypi/templates/package.html @@ -7,8 +7,8 @@

Links for {{ package.name }}

{% for release in package.releases %} - {{ release }} + {{ release }}
{% endfor %} - \ No newline at end of file + From 74de4d6af092ee21d49c5c58138133830eb5d8a8 Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Wed, 11 Jul 2018 15:59:39 -0500 Subject: [PATCH 09/27] Remove duplicate dir check --- tests/pytest-pypi/pytest_pypi/app.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/pytest-pypi/pytest_pypi/app.py b/tests/pytest-pypi/pytest_pypi/app.py index 7ac1bc4b..ba1a5437 100644 --- a/tests/pytest-pypi/pytest_pypi/app.py +++ b/tests/pytest-pypi/pytest_pypi/app.py @@ -41,8 +41,6 @@ class Package(object): def prepare_packages(path): """Add packages in path to the registry.""" path = os.path.abspath(path) - if not (os.path.exists(path) and os.path.isdir(path)): - raise ValueError("{} is not a directory!".format(path)) if not (os.path.exists(path) and os.path.isdir(path)): raise ValueError("{} is not a directory!".format(path)) for root, dirs, files in os.walk(path): From 1ceccb9bdf20d7b54913a886a3b6273c40366e1c Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Wed, 11 Jul 2018 23:32:43 -0500 Subject: [PATCH 10/27] Fix extras check in test --- tests/integration/test_lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 67da46b9..25195bfb 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -363,7 +363,7 @@ requests = {git = "https://github.com/requests/requests.git", editable = true, e assert 'requests' in p.lockfile['default'] assert 'idna' in p.lockfile['default'] assert 'chardet' in p.lockfile['default'] - assert 'pysocks' in p.lockfile['default'] + assert "socks" in p.lockfile["default"]["requests"]["extras"] c = p.pipenv('install') assert c.return_code == 0 From 05b35817566c888b84944ad4acefdc66b98e6300 Mon Sep 17 00:00:00 2001 From: Erin O'Connell Date: Thu, 12 Jul 2018 02:13:20 -0600 Subject: [PATCH 11/27] Revert "Add docstrings to show short_help when running --help" --- pipenv/cli.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 1b774508..fb0b2b2d 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -394,7 +394,6 @@ def install( keep_outdated=False, selective_upgrade=False, ): - """Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.""" from .core import do_install do_install( @@ -483,7 +482,6 @@ def uninstall( keep_outdated=False, pypi_mirror=None, ): - """Un-installs a provided package and removes it from Pipfile.""" from .core import do_uninstall do_uninstall( @@ -563,7 +561,6 @@ def lock( pre=False, keep_outdated=False, ): - """Generates Pipfile.lock.""" from .core import ensure_project, do_init, do_lock # Ensure that virtualenv is available. @@ -624,7 +621,6 @@ def shell( anyway=False, pypi_mirror=None, ): - """Spawns a shell within the virtualenv.""" from .core import load_dot_env, do_shell # Prevent user from activating nested environments. @@ -687,7 +683,6 @@ def shell( help="Specify a PyPI mirror.", ) def run(command, args, three=None, python=False, pypi_mirror=None): - """Spawns a command installed into the virtualenv.""" from .core import do_run do_run( @@ -743,7 +738,6 @@ def check( args=None, pypi_mirror=None, ): - """Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.""" from .core import do_check do_check( @@ -833,7 +827,6 @@ def update( outdated=False, more_packages=None, ): - """Runs lock, then sync.""" from .core import ( ensure_project, do_outdated, @@ -898,7 +891,6 @@ def update( @option("--json-tree", is_flag=True, default=False, help="Output JSON in nested tree.") @option("--reverse", is_flag=True, default=False, help="Reversed dependency graph.") def graph(bare=False, json=False, json_tree=False, reverse=False): - """Displays currently-installed dependency graph information.""" from .core import do_graph do_graph(bare=bare, json=json, json_tree=json_tree, reverse=reverse) @@ -927,7 +919,6 @@ def graph(bare=False, json=False, json_tree=False, reverse=False): ) @argument("module", nargs=1) def run_open(module, three=None, python=None, pypi_mirror=None): - """View a given module in your editor.""" from .core import which, ensure_project # Ensure that virtualenv is available. @@ -1009,7 +1000,6 @@ def sync( sequential=False, pypi_mirror=None, ): - """Installs all packages specified in Pipfile.lock.""" from .core import do_sync do_sync( @@ -1055,7 +1045,6 @@ def sync( def clean( ctx, three=None, python=None, dry_run=False, bare=False, user=False, verbose=False ): - """Uninstalls all packages not specified in Pipfile.lock.""" from .core import do_clean do_clean(ctx=ctx, three=three, python=python, dry_run=dry_run, verbose=verbose) From dbc9008131f5c9684e6b5336a4ca2993f518d9a6 Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Thu, 12 Jul 2018 08:05:58 -0500 Subject: [PATCH 12/27] Duck type prettytoml types during serialization --- pipenv/project.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index d0d21c3e..918c7184 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -14,7 +14,6 @@ import pipfile.api import six import toml import json as simplejson -from prettytoml.elements.array import ArrayElement from ._compat import Path @@ -66,7 +65,7 @@ DEFAULT_NEWLINES = u"\n" def encode_toml_elements(obj): - if isinstance(obj, ArrayElement): + if hasattr(obj, 'primitive_value'): return obj.primitive_value raise TypeError(repr(obj) + " is not JSON serializable") From f71167853737ac2f22166d5e9529e18ee92f9a7c Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 13 Jul 2018 03:14:21 +0800 Subject: [PATCH 13/27] Get virtualenv location from output The Project class's implementation is a fucking mess. Avoid that. --- tests/integration/test_install_basic.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index e5a48127..ab6648f4 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -359,10 +359,16 @@ def test_install_venv_project_directory(PipenvInstance, pypi): os.environ["WORKON_HOME"] = workon_home.name if "PIPENV_VENV_IN_PROJECT" in os.environ: del os.environ["PIPENV_VENV_IN_PROJECT"] + c = p.pipenv("install six") assert c.return_code == 0 - project = Project() - assert Path(project.virtualenv_location).joinpath(".project").exists() + + venv_loc = None + for line in c.err.splitlines(): + if line.startswith("Virtualenv location:"): + venv_loc = Path(line.split(":", 1)[-1].strip()) + assert venv_loc is not None + assert venv_loc.joinpath(".project").exists() @pytest.mark.deploy From 8f2b4abeb116fa5375f62d63239c3cefc405428b Mon Sep 17 00:00:00 2001 From: Charles Lai Date: Thu, 12 Jul 2018 16:05:20 -0400 Subject: [PATCH 14/27] Revert "Revert "Add docstrings to show short_help when running --help"" This reverts commit 05b35817566c888b84944ad4acefdc66b98e6300. --- pipenv/cli.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pipenv/cli.py b/pipenv/cli.py index fb0b2b2d..1b774508 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -394,6 +394,7 @@ def install( keep_outdated=False, selective_upgrade=False, ): + """Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.""" from .core import do_install do_install( @@ -482,6 +483,7 @@ def uninstall( keep_outdated=False, pypi_mirror=None, ): + """Un-installs a provided package and removes it from Pipfile.""" from .core import do_uninstall do_uninstall( @@ -561,6 +563,7 @@ def lock( pre=False, keep_outdated=False, ): + """Generates Pipfile.lock.""" from .core import ensure_project, do_init, do_lock # Ensure that virtualenv is available. @@ -621,6 +624,7 @@ def shell( anyway=False, pypi_mirror=None, ): + """Spawns a shell within the virtualenv.""" from .core import load_dot_env, do_shell # Prevent user from activating nested environments. @@ -683,6 +687,7 @@ def shell( help="Specify a PyPI mirror.", ) def run(command, args, three=None, python=False, pypi_mirror=None): + """Spawns a command installed into the virtualenv.""" from .core import do_run do_run( @@ -738,6 +743,7 @@ def check( args=None, pypi_mirror=None, ): + """Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.""" from .core import do_check do_check( @@ -827,6 +833,7 @@ def update( outdated=False, more_packages=None, ): + """Runs lock, then sync.""" from .core import ( ensure_project, do_outdated, @@ -891,6 +898,7 @@ def update( @option("--json-tree", is_flag=True, default=False, help="Output JSON in nested tree.") @option("--reverse", is_flag=True, default=False, help="Reversed dependency graph.") def graph(bare=False, json=False, json_tree=False, reverse=False): + """Displays currently-installed dependency graph information.""" from .core import do_graph do_graph(bare=bare, json=json, json_tree=json_tree, reverse=reverse) @@ -919,6 +927,7 @@ def graph(bare=False, json=False, json_tree=False, reverse=False): ) @argument("module", nargs=1) def run_open(module, three=None, python=None, pypi_mirror=None): + """View a given module in your editor.""" from .core import which, ensure_project # Ensure that virtualenv is available. @@ -1000,6 +1009,7 @@ def sync( sequential=False, pypi_mirror=None, ): + """Installs all packages specified in Pipfile.lock.""" from .core import do_sync do_sync( @@ -1045,6 +1055,7 @@ def sync( def clean( ctx, three=None, python=None, dry_run=False, bare=False, user=False, verbose=False ): + """Uninstalls all packages not specified in Pipfile.lock.""" from .core import do_clean do_clean(ctx=ctx, three=three, python=python, dry_run=dry_run, verbose=verbose) From 98a52149d7a85419f3067fa3cba2fe1deb59e705 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 13 Jul 2018 15:03:59 +0800 Subject: [PATCH 15/27] Seperate lock file concerns in a custom encoder --- pipenv/project.py | 51 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 918c7184..1dc2aaa4 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -13,7 +13,6 @@ import pipfile import pipfile.api import six import toml -import json as simplejson from ._compat import Path @@ -64,16 +63,36 @@ def _normalized(p): DEFAULT_NEWLINES = u"\n" -def encode_toml_elements(obj): - if hasattr(obj, 'primitive_value'): - return obj.primitive_value - raise TypeError(repr(obj) + " is not JSON serializable") +class _LockFileEncoder(json.JSONEncoder): + """A specilized JSON encoder to convert loaded TOML data into a lock file. + + This adds a few characteristics to the encoder: + + * The JSON is always prettified with indents and spaces. + * PrettyTOML's container elements are seamlessly encodable. + * The output is always UTF-8-encoded text, never binary, even on Python 2. + """ + def __init__(self): + super(_LockFileEncoder, self).__init__( + indent=4, separators=(",", ": "), sort_keys=True, + ) + + def default(self, obj): + from prettytoml.elements.common import ContainerElement + if isinstance(obj, ContainerElement): + return obj.primitive_value + return super(_LockFileEncoder, self).default(obj) + + def encode(self, obj): + content = super(_LockFileEncoder, self).encode(obj) + if not isinstance(content, six.text_type): + content = content.decode("utf-8") + return content def preferred_newlines(f): if isinstance(f.newlines, six.text_type): return f.newlines - return DEFAULT_NEWLINES @@ -111,6 +130,8 @@ class SourceNotFound(KeyError): class Project(object): """docstring for Project""" + _lockfile_encoder = _LockFileEncoder() + def __init__(self, which=None, python_version=None, chdir=True): super(Project, self).__init__() self._name = None @@ -635,15 +656,17 @@ class Project(object): def write_lockfile(self, content): """Write out the lockfile. """ - newlines = self._lockfile_newlines - s = simplejson.dumps( # Send Unicode in to guarentee Unicode out. - content, indent=4, separators=(u",", u": "), sort_keys=True, - default=encode_toml_elements, - ) - with atomic_open_for_write(self.lockfile_location, newline=newlines) as f: + s = self._lockfile_encoder.encode(content) + open_kwargs = { + 'newline': self._lockfile_newlines, + 'encoding': 'utf-8', + } + with atomic_open_for_write(self.lockfile_location, **open_kwargs) as f: f.write(s) + # Write newline at end of document. GH-319. + # Only need '\n' here; the file object handles the rest. if not s.endswith(u"\n"): - f.write(u"\n") # Write newline at end of document. GH #319. + f.write(u"\n") @property def pipfile_sources(self): @@ -758,7 +781,7 @@ class Project(object): self.write_toml(self.parsed_pipfile) def load_lockfile(self, expand_env_vars=True): - with io.open(self.lockfile_location) as lock: + with io.open(self.lockfile_location, encoding='utf-8') as lock: j = json.load(lock) self._lockfile_newlines = preferred_newlines(lock) # lockfile is just a string From 36239140017466baf2956fa1da2d640382d7ff82 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 13 Jul 2018 22:26:43 +0800 Subject: [PATCH 16/27] Also allow TokenElement in lock file encoder Can't be too careful hadling things here. --- pipenv/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 1dc2aaa4..79fa1b91 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -78,8 +78,8 @@ class _LockFileEncoder(json.JSONEncoder): ) def default(self, obj): - from prettytoml.elements.common import ContainerElement - if isinstance(obj, ContainerElement): + from prettytoml.elements.common import ContainerElement, TokenElement + if isinstance(obj, (ContainerElement, TokenElement)): return obj.primitive_value return super(_LockFileEncoder, self).default(obj) From 2355bdca00f98f8dda14df8ec2e8e16a5b3f6373 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 16:14:32 -0400 Subject: [PATCH 17/27] Update release task Signed-off-by: Dan Ryan --- tasks/release.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tasks/release.py b/tasks/release.py index c1946fa7..d2c6e1f1 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -91,16 +91,24 @@ def bump_version(ctx, dry_run=False, increment=True, release=False, dev=False, p if pre and not tag: print('Using "pre" requires a corresponding tag.') return - if release and not dev and not pre: + if release and not dev and not pre and increment: new_version = current_version.replace(release=today.timetuple()[:3]).clear(pre=True, dev=True) elif release and (dev or pre): - new_version = current_version.replace(release=today.timetuple()[:3]) + if increment: + new_version = current_version.replace(release=today.timetuple()[:3]) + else: + new_version = current_version if dev: new_version = new_version.bump_dev() elif pre: new_version = new_version.bump_pre(tag=tag) else: - new_version = current_version.replace(release=next_month) + if not release: + increment = False + if increment: + new_version = current_version.replace(release=next_month) + else: + new_version = current_version if dev: new_version = new_version.bump_dev() elif pre: From 6ead6ff8bc8013da9b93de4f95fd0e1278ee39e7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 16:18:11 -0400 Subject: [PATCH 18/27] Bumped version. Signed-off-by: Dan Ryan --- pipenv/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/__version__.py b/pipenv/__version__.py index 8c8673ae..104f7941 100644 --- a/pipenv/__version__.py +++ b/pipenv/__version__.py @@ -2,4 +2,4 @@ # // ) ) / / // ) ) //___) ) // ) ) || / / # //___/ / / / //___/ / // // / / || / / # // / / // ((____ // / / ||/ / -__version__ = "2018.7.1" +__version__ = "2018.7.1.dev0" From d8b6ccaed7d180a922bbcafeeb3dcda569f73dda Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 16:18:32 -0400 Subject: [PATCH 19/27] Added missing news entries Signed-off-by: Dan Ryan --- news/2542.behavior | 1 + news/2561.bugfix | 1 + news/2568.doc | 2 ++ news/2568.feature | 2 ++ 4 files changed, 6 insertions(+) create mode 100644 news/2542.behavior create mode 100644 news/2561.bugfix create mode 100644 news/2568.doc create mode 100644 news/2568.feature diff --git a/news/2542.behavior b/news/2542.behavior new file mode 100644 index 00000000..245bae63 --- /dev/null +++ b/news/2542.behavior @@ -0,0 +1 @@ +``pipenv --help`` will now include short help descriptions. diff --git a/news/2561.bugfix b/news/2561.bugfix new file mode 100644 index 00000000..98f1738b --- /dev/null +++ b/news/2561.bugfix @@ -0,0 +1 @@ +Fixed a bug which sometimes caused pipenv to throw a ``TypeError`` or to run into encoding issues when writing lockfiles on python 2. diff --git a/news/2568.doc b/news/2568.doc new file mode 100644 index 00000000..ab2644ff --- /dev/null +++ b/news/2568.doc @@ -0,0 +1,2 @@ +Simplified the test configuration process. + diff --git a/news/2568.feature b/news/2568.feature new file mode 100644 index 00000000..47972186 --- /dev/null +++ b/news/2568.feature @@ -0,0 +1,2 @@ +Updated test-pypi addon to better support json-api access (forward compatibility). +Improved testing process for new contributors. From 6b1ec91607826e463df3b46fc86652704cf22936 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 16:22:06 -0400 Subject: [PATCH 20/27] Added working draft of changelog and release task for markdown conversion Signed-off-by: Dan Ryan --- CHANGELOG.draft.rst | 50 +++++++++++++++++++++++++++++++++++++++++++++ tasks/release.py | 8 +++++++- 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.draft.rst diff --git a/CHANGELOG.draft.rst b/CHANGELOG.draft.rst new file mode 100644 index 00000000..f861b218 --- /dev/null +++ b/CHANGELOG.draft.rst @@ -0,0 +1,50 @@ +2018.7.1.dev0 (2018-07-15) +========================== + + +Features & Improvements +----------------------- + +- Updated test-pypi addon to better support json-api access (forward compatibility). + Improved testing process for new contributors. `#2568 `_ + + +Behavior Changes +---------------- + +- Virtual environment activation for ``run`` is revised to improve interpolation + with other Python discovery tools. `#2503 `_ + +- Improve terminal coloring to display better in Powershell. `#2511 `_ + +- Invoke ``virtualenv`` directly for virtual environment creation, instead of depending on ``pew``. `#2518 `_ + +- ``pipenv --help`` will now include short help descriptions. `#2542 `_ + + +Bug Fixes +--------- + +- Fix subshell invocation on Windows for Python 2. `#2515 `_ + +- Fixed a bug which sometimes caused pipenv to throw a ``TypeError`` or to run into encoding issues when writing lockfiles on python 2. `#2561 `_ + +- Improve quoting logic for ``pipenv run`` so it works better with Windows + built-in commands. `#2563 `_ + +- Fixed a bug related to parsing vcs requirements with both extras and subdirectory fragments. + Corrected an issue in the ``requirementslib`` parser which led to some markers being discarded rather than evaluated. `#2564 `_ + + +Vendored Libraries +------------------ + +- Pew is no longer vendored. Entry point ``pewtwo``, packages ``pipenv.pew`` and + ``pipenv.patched.pew`` are removed. `#2521 `_ + + +Improved Documentation +---------------------- + +- Simplified the test configuration process. `#2568 `_ + diff --git a/tasks/release.py b/tasks/release.py index d2c6e1f1..4a242ba5 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -41,7 +41,7 @@ def drop_dist_dirs(ctx): def build_dists(ctx): drop_dist_dirs(ctx) log('Building sdist using %s ....' % sys.executable) - for py_version in ['2.7', '3.6']: + for py_version in ['2.7', '3.6', '3.7']: env = {'PIPENV_PYTHON': py_version} ctx.run('pipenv install --dev', env=env) if py_version == '3.6': @@ -56,6 +56,12 @@ def upload_dists(ctx): ctx.run('twine upload dist/*') +@invoke.task +def generate_markdown(ctx): + log('Generating markdown from changelog...') + ctx.run('pandoc CHANGELOG.rst -f rst -t markdown -o CHANGELOG.md') + + @invoke.task def generate_changelog(ctx, commit=False, draft=False): log('Generating changelog...') From ff3c49e5b43f2eda6b026b8d5ad6e30f635df124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20G=C3=B3mez=20Mart=C3=ADn?= Date: Mon, 16 Jul 2018 15:33:57 +0200 Subject: [PATCH 21/27] Add pkg-resources to the ignored packages list --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 8c21468c..337b7c72 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -64,7 +64,7 @@ from .environments import ( ) # Packages that should be ignored later. -BAD_PACKAGES = ("setuptools", "pip", "wheel", "packaging", "distribute") +BAD_PACKAGES = ("distribute", "packaging", "pip", "pkg-resources", "setuptools", "wheel") # Are we using the default Python? USING_DEFAULT_PYTHON = True if not PIPENV_HIDE_EMOJIS: From 4f433c01316e47189ef25e8afe7dfcf66516354c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20G=C3=B3mez=20Mart=C3=ADn?= Date: Mon, 16 Jul 2018 15:58:52 +0200 Subject: [PATCH 22/27] Fix broken link in CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bb17336e..6e3ef65c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,7 +70,7 @@ dependencies. Once that has completed, you can start developing. The repo version of Pipenv must be installed over other global versions to resolve conflicts with the `pipenv` folder being implicitly added to `sys.path`. -See pypa/pipenv#2557 for more details. +See [pypa/pipenv#2557](https://github.com/pypa/pipenv/issues/2557) for more details. ### Testing From fe89ed43ec8d528f7c42500685f9b587e53f3ae3 Mon Sep 17 00:00:00 2001 From: Giuseppe Date: Tue, 17 Jul 2018 12:42:55 +0200 Subject: [PATCH 23/27] Add --rm option example to Usage Examples --- README.md | 3 +++ pipenv/core.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index c674d80c..7ff9cd14 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,9 @@ Fish is the best shell. You should use it. Create a new project using Python 3.7, specifically: $ pipenv --python 3.7 + Remove project virtualenv (inferred from current directory): + $ pipenv --rm + Install all dependencies for a project (including dev): $ pipenv install --dev diff --git a/pipenv/core.py b/pipenv/core.py index 337b7c72..a678a3b2 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1560,6 +1560,9 @@ Usage Examples: Create a new project using Python 3.7, specifically: $ {1} + Remove project virtualenv (inferred from current directory): + $ {9} + Install all dependencies for a project (including dev): $ {2} @@ -1588,6 +1591,7 @@ Commands:""".format( crayons.red("pipenv lock --pre"), crayons.red("pipenv check"), crayons.red("pipenv run pip freeze"), + crayons.red("pipenv --rm"), ) help = help.replace("Commands:", additional_help) return help From f141911dabe725c9e6403647e4c46e914b7bf095 Mon Sep 17 00:00:00 2001 From: Nitesh Sharma Date: Tue, 17 Jul 2018 13:58:11 +0530 Subject: [PATCH 24/27] Show better error if virtual env is broken Closes https://github.com/pypa/pipenv/issues/1918 --- pipenv/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index a678a3b2..a6372ad2 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -636,12 +636,15 @@ def ensure_project( ): click.echo( "{0}: Your Pipfile requires {1} {2}, " - "but you are using {3} ({4}).".format( + "but you are using {3} ({4}). Running" + "{5} and rebuild the virtual environment" + "may resolve the issue".format( crayons.red("Warning", bold=True), crayons.normal("python_version", bold=True), crayons.blue(project.required_python_version), crayons.blue(python_version(path_to_python)), crayons.green(shorten_path(path_to_python)), + crayons.green("`pipenv --rm`") ), err=True, ) From 72d643c1f27791dfad8ed2e441da4374ac95735e Mon Sep 17 00:00:00 2001 From: Charles Lai Date: Tue, 17 Jul 2018 17:06:21 -0700 Subject: [PATCH 25/27] Make README badges inline --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 7ff9cd14..9c2f00b3 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,9 @@ Pipenv: Python Development Workflow for Humans ============================================== [![image](https://img.shields.io/pypi/v/pipenv.svg)](https://pypi.python.org/pypi/pipenv) - [![image](https://img.shields.io/pypi/l/pipenv.svg)](https://pypi.python.org/pypi/pipenv) - [![image](https://badge.buildkite.com/79c7eccf056b17c3151f3c4d0e4c4b8b724539d84f1e037b9b.svg?branch=master)](https://code.kennethreitz.org/source/pipenv/) - [![image](https://img.shields.io/pypi/pyversions/pipenv.svg)](https://pypi.python.org/pypi/pipenv) - [![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz) ------------------------------------------------------------------------ From b44082115314113668ee700f2598324309a92916 Mon Sep 17 00:00:00 2001 From: Rod Xavier Bondoc Date: Wed, 18 Jul 2018 16:24:16 +1000 Subject: [PATCH 26/27] Pass PIPENV_DOTENV_LOCATION directly to load_dotenv if present --- pipenv/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index a6372ad2..dc4d2acc 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -150,8 +150,8 @@ def load_dot_env(): if not PIPENV_DONT_LOAD_ENV: # If the project doesn't exist yet, check current directory for a .env file project_directory = project.project_directory or "." - denv = dotenv.find_dotenv( - PIPENV_DOTENV_LOCATION or os.sep.join([project_directory, ".env"]) + denv = PIPENV_DOTENV_LOCATION or dotenv.find_dotenv( + os.sep.join([project_directory, ".env"]) ) if os.path.isfile(denv): click.echo( From 5cbe272550b635f9cdd8f9789c918f911ec60eca Mon Sep 17 00:00:00 2001 From: Rod Xavier Bondoc Date: Wed, 18 Jul 2018 16:45:44 +1000 Subject: [PATCH 27/27] Remove find_dotenv() --- pipenv/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index dc4d2acc..af4c9761 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -150,9 +150,8 @@ def load_dot_env(): if not PIPENV_DONT_LOAD_ENV: # If the project doesn't exist yet, check current directory for a .env file project_directory = project.project_directory or "." - denv = PIPENV_DOTENV_LOCATION or dotenv.find_dotenv( - os.sep.join([project_directory, ".env"]) - ) + denv = PIPENV_DOTENV_LOCATION or os.sep.join([project_directory, ".env"]) + if os.path.isfile(denv): click.echo( crayons.normal("Loading .env environment variables…", bold=True),