diff --git a/.env b/.env index 189ddda7..d77e8f12 100644 --- a/.env +++ b/.env @@ -1,2 +1 @@ HELLO=WORLD -PYPI_VENDOR_DIR="./tests/pypi/" \ No newline at end of file diff --git a/CHANGELOG.draft.rst b/CHANGELOG.draft.rst new file mode 100644 index 00000000..f861b218 --- /dev/null +++ b/CHANGELOG.draft.rst @@ -0,0 +1,50 @@ +2018.7.1.dev0 (2018-07-15) +========================== + + +Features & Improvements +----------------------- + +- Updated test-pypi addon to better support json-api access (forward compatibility). + Improved testing process for new contributors. `#2568 `_ + + +Behavior Changes +---------------- + +- Virtual environment activation for ``run`` is revised to improve interpolation + with other Python discovery tools. `#2503 `_ + +- Improve terminal coloring to display better in Powershell. `#2511 `_ + +- Invoke ``virtualenv`` directly for virtual environment creation, instead of depending on ``pew``. `#2518 `_ + +- ``pipenv --help`` will now include short help descriptions. `#2542 `_ + + +Bug Fixes +--------- + +- Fix subshell invocation on Windows for Python 2. `#2515 `_ + +- Fixed a bug which sometimes caused pipenv to throw a ``TypeError`` or to run into encoding issues when writing lockfiles on python 2. `#2561 `_ + +- Improve quoting logic for ``pipenv run`` so it works better with Windows + built-in commands. `#2563 `_ + +- Fixed a bug related to parsing vcs requirements with both extras and subdirectory fragments. + Corrected an issue in the ``requirementslib`` parser which led to some markers being discarded rather than evaluated. `#2564 `_ + + +Vendored Libraries +------------------ + +- Pew is no longer vendored. Entry point ``pewtwo``, packages ``pipenv.pew`` and + ``pipenv.patched.pew`` are removed. `#2521 `_ + + +Improved Documentation +---------------------- + +- Simplified the test configuration process. `#2568 `_ + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7fa45c6..6e3ef65c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,3 +55,45 @@ Please be aware of the following things when filing bug reports: If you do not provide all of these things, it will take us much longer to fix your problem. If we ask you to clarify these and you never respond, we will close your issue without fixing it. + +## Development Setup + +To get your development environment setup, run: + +```sh +pip install -e . +pipenv install --dev +``` + +This will install the repo version of Pipenv and then install the development +dependencies. Once that has completed, you can start developing. + +The repo version of Pipenv must be installed over other global versions to +resolve conflicts with the `pipenv` folder being implicitly added to `sys.path`. +See [pypa/pipenv#2557](https://github.com/pypa/pipenv/issues/2557) for more details. + +### Testing + +Tests are written in `pytest` style and can be run very simply: + +```sh +pytest +``` + +This will run all Pipenv tests, which can take awhile. To run a subset of the +tests, the standard pytest filters are available, such as: + +- provide a directory or file: `pytest tests/unit` or `pytest tests/unit/test_cmdparse.py` +- provide a keyword expression: `pytest -k test_lock_editable_vcs_without_install` +- provide a nodeid: `pytest tests/unit/test_cmdparse.py::test_parse` +- provide a test marker: `pytest -m lock` + +#### Package Index + +To speed up testing, tests that rely on a package index for locking and +installing use a local server that contains vendored packages in the +`tests/pypi` directory. Each vendored package should have it's own folder +containing the necessary releases. When adding a release for a package, it is +easiest to use either the `.tar.gz` or universal wheels (ex: `py2.py3-none`). If +a `.tar.gz` or universal wheel is not available, add wheels for all available +architectures and platforms. diff --git a/README.md b/README.md index c674d80c..9c2f00b3 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,9 @@ Pipenv: Python Development Workflow for Humans ============================================== [![image](https://img.shields.io/pypi/v/pipenv.svg)](https://pypi.python.org/pypi/pipenv) - [![image](https://img.shields.io/pypi/l/pipenv.svg)](https://pypi.python.org/pypi/pipenv) - [![image](https://badge.buildkite.com/79c7eccf056b17c3151f3c4d0e4c4b8b724539d84f1e037b9b.svg?branch=master)](https://code.kennethreitz.org/source/pipenv/) - [![image](https://img.shields.io/pypi/pyversions/pipenv.svg)](https://pypi.python.org/pypi/pipenv) - [![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz) ------------------------------------------------------------------------ @@ -157,6 +153,9 @@ Fish is the best shell. You should use it. Create a new project using Python 3.7, specifically: $ pipenv --python 3.7 + Remove project virtualenv (inferred from current directory): + $ pipenv --rm + Install all dependencies for a project (including dev): $ pipenv install --dev diff --git a/news/2542.behavior b/news/2542.behavior new file mode 100644 index 00000000..245bae63 --- /dev/null +++ b/news/2542.behavior @@ -0,0 +1 @@ +``pipenv --help`` will now include short help descriptions. diff --git a/news/2561.bugfix b/news/2561.bugfix new file mode 100644 index 00000000..98f1738b --- /dev/null +++ b/news/2561.bugfix @@ -0,0 +1 @@ +Fixed a bug which sometimes caused pipenv to throw a ``TypeError`` or to run into encoding issues when writing lockfiles on python 2. diff --git a/news/2568.doc b/news/2568.doc new file mode 100644 index 00000000..ab2644ff --- /dev/null +++ b/news/2568.doc @@ -0,0 +1,2 @@ +Simplified the test configuration process. + diff --git a/news/2568.feature b/news/2568.feature new file mode 100644 index 00000000..47972186 --- /dev/null +++ b/news/2568.feature @@ -0,0 +1,2 @@ +Updated test-pypi addon to better support json-api access (forward compatibility). +Improved testing process for new contributors. diff --git a/pipenv/__version__.py b/pipenv/__version__.py index 8c8673ae..104f7941 100644 --- a/pipenv/__version__.py +++ b/pipenv/__version__.py @@ -2,4 +2,4 @@ # // ) ) / / // ) ) //___) ) // ) ) || / / # //___/ / / / //___/ / // // / / || / / # // / / // ((____ // / / ||/ / -__version__ = "2018.7.1" +__version__ = "2018.7.1.dev0" diff --git a/pipenv/cli.py b/pipenv/cli.py index fb0b2b2d..1b774508 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -394,6 +394,7 @@ def install( keep_outdated=False, selective_upgrade=False, ): + """Installs provided packages and adds them to Pipfile, or (if none is given), installs all packages.""" from .core import do_install do_install( @@ -482,6 +483,7 @@ def uninstall( keep_outdated=False, pypi_mirror=None, ): + """Un-installs a provided package and removes it from Pipfile.""" from .core import do_uninstall do_uninstall( @@ -561,6 +563,7 @@ def lock( pre=False, keep_outdated=False, ): + """Generates Pipfile.lock.""" from .core import ensure_project, do_init, do_lock # Ensure that virtualenv is available. @@ -621,6 +624,7 @@ def shell( anyway=False, pypi_mirror=None, ): + """Spawns a shell within the virtualenv.""" from .core import load_dot_env, do_shell # Prevent user from activating nested environments. @@ -683,6 +687,7 @@ def shell( help="Specify a PyPI mirror.", ) def run(command, args, three=None, python=False, pypi_mirror=None): + """Spawns a command installed into the virtualenv.""" from .core import do_run do_run( @@ -738,6 +743,7 @@ def check( args=None, pypi_mirror=None, ): + """Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.""" from .core import do_check do_check( @@ -827,6 +833,7 @@ def update( outdated=False, more_packages=None, ): + """Runs lock, then sync.""" from .core import ( ensure_project, do_outdated, @@ -891,6 +898,7 @@ def update( @option("--json-tree", is_flag=True, default=False, help="Output JSON in nested tree.") @option("--reverse", is_flag=True, default=False, help="Reversed dependency graph.") def graph(bare=False, json=False, json_tree=False, reverse=False): + """Displays currently-installed dependency graph information.""" from .core import do_graph do_graph(bare=bare, json=json, json_tree=json_tree, reverse=reverse) @@ -919,6 +927,7 @@ def graph(bare=False, json=False, json_tree=False, reverse=False): ) @argument("module", nargs=1) def run_open(module, three=None, python=None, pypi_mirror=None): + """View a given module in your editor.""" from .core import which, ensure_project # Ensure that virtualenv is available. @@ -1000,6 +1009,7 @@ def sync( sequential=False, pypi_mirror=None, ): + """Installs all packages specified in Pipfile.lock.""" from .core import do_sync do_sync( @@ -1045,6 +1055,7 @@ def sync( def clean( ctx, three=None, python=None, dry_run=False, bare=False, user=False, verbose=False ): + """Uninstalls all packages not specified in Pipfile.lock.""" from .core import do_clean do_clean(ctx=ctx, three=three, python=python, dry_run=dry_run, verbose=verbose) diff --git a/pipenv/core.py b/pipenv/core.py index d8f28f64..ffe86cfa 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -64,7 +64,7 @@ from .environments import ( ) # Packages that should be ignored later. -BAD_PACKAGES = ("setuptools", "pip", "wheel", "packaging", "distribute") +BAD_PACKAGES = ("distribute", "packaging", "pip", "pkg-resources", "setuptools", "wheel") # Are we using the default Python? USING_DEFAULT_PYTHON = True if not PIPENV_HIDE_EMOJIS: @@ -150,9 +150,8 @@ def load_dot_env(): if not PIPENV_DONT_LOAD_ENV: # If the project doesn't exist yet, check current directory for a .env file project_directory = project.project_directory or "." - denv = dotenv.find_dotenv( - PIPENV_DOTENV_LOCATION or os.sep.join([project_directory, ".env"]) - ) + denv = PIPENV_DOTENV_LOCATION or os.sep.join([project_directory, ".env"]) + if os.path.isfile(denv): click.echo( crayons.normal("Loading .env environment variables…", bold=True), @@ -617,12 +616,15 @@ def ensure_project( ): click.echo( "{0}: Your Pipfile requires {1} {2}, " - "but you are using {3} ({4}).".format( + "but you are using {3} ({4}). Running" + "{5} and rebuild the virtual environment" + "may resolve the issue".format( crayons.red("Warning", bold=True), crayons.normal("python_version", bold=True), crayons.blue(project.required_python_version), crayons.blue(python_version(path_to_python)), crayons.green(shorten_path(path_to_python)), + crayons.green("`pipenv --rm`") ), err=True, ) @@ -1541,6 +1543,9 @@ Usage Examples: Create a new project using Python 3.7, specifically: $ {1} + Remove project virtualenv (inferred from current directory): + $ {9} + Install all dependencies for a project (including dev): $ {2} @@ -1569,6 +1574,7 @@ Commands:""".format( crayons.red("pipenv lock --pre"), crayons.red("pipenv check"), crayons.red("pipenv run pip freeze"), + crayons.red("pipenv --rm"), ) help = help.replace("Commands:", additional_help) return help diff --git a/pipenv/project.py b/pipenv/project.py index 7f40576d..79fa1b91 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -13,7 +13,6 @@ import pipfile import pipfile.api import six import toml -import json as simplejson from ._compat import Path @@ -64,10 +63,36 @@ def _normalized(p): DEFAULT_NEWLINES = u"\n" +class _LockFileEncoder(json.JSONEncoder): + """A specilized JSON encoder to convert loaded TOML data into a lock file. + + This adds a few characteristics to the encoder: + + * The JSON is always prettified with indents and spaces. + * PrettyTOML's container elements are seamlessly encodable. + * The output is always UTF-8-encoded text, never binary, even on Python 2. + """ + def __init__(self): + super(_LockFileEncoder, self).__init__( + indent=4, separators=(",", ": "), sort_keys=True, + ) + + def default(self, obj): + from prettytoml.elements.common import ContainerElement, TokenElement + if isinstance(obj, (ContainerElement, TokenElement)): + return obj.primitive_value + return super(_LockFileEncoder, self).default(obj) + + def encode(self, obj): + content = super(_LockFileEncoder, self).encode(obj) + if not isinstance(content, six.text_type): + content = content.decode("utf-8") + return content + + def preferred_newlines(f): if isinstance(f.newlines, six.text_type): return f.newlines - return DEFAULT_NEWLINES @@ -105,6 +130,8 @@ class SourceNotFound(KeyError): class Project(object): """docstring for Project""" + _lockfile_encoder = _LockFileEncoder() + def __init__(self, which=None, python_version=None, chdir=True): super(Project, self).__init__() self._name = None @@ -629,14 +656,17 @@ class Project(object): def write_lockfile(self, content): """Write out the lockfile. """ - newlines = self._lockfile_newlines - s = simplejson.dumps( # Send Unicode in to guarentee Unicode out. - content, indent=4, separators=(u",", u": "), sort_keys=True - ) - with atomic_open_for_write(self.lockfile_location, newline=newlines) as f: + s = self._lockfile_encoder.encode(content) + open_kwargs = { + 'newline': self._lockfile_newlines, + 'encoding': 'utf-8', + } + with atomic_open_for_write(self.lockfile_location, **open_kwargs) as f: f.write(s) + # Write newline at end of document. GH-319. + # Only need '\n' here; the file object handles the rest. if not s.endswith(u"\n"): - f.write(u"\n") # Write newline at end of document. GH #319. + f.write(u"\n") @property def pipfile_sources(self): @@ -751,7 +781,7 @@ class Project(object): self.write_toml(self.parsed_pipfile) def load_lockfile(self, expand_env_vars=True): - with io.open(self.lockfile_location) as lock: + with io.open(self.lockfile_location, encoding='utf-8') as lock: j = json.load(lock) self._lockfile_newlines = preferred_newlines(lock) # lockfile is just a string diff --git a/pytest.ini b/pytest.ini index 92e72fd5..80c3e3a3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,4 @@ [pytest] addopts = -n auto -norecursedirs = vendor patched +; Add vendor and patched in addition to the default list of ignored dirs +norecursedirs = .* build dist CVS _darcs {arch} *.egg vendor patched diff --git a/run-tests.bat b/run-tests.bat index 4ddcee2b..f31a562a 100644 --- a/run-tests.bat +++ b/run-tests.bat @@ -4,4 +4,4 @@ virtualenv R:\.venv R:\.venv\Scripts\pip install -e . --upgrade --upgrade-strategy=only-if-needed R:\.venv\Scripts\pipenv install --dev -SET RAM_DISK=R:&& SET PYPI_VENDOR_DIR=".\tests\pypi\" && R:\.venv\Scripts\pipenv run pytest -n auto -v tests --tap-stream > report.tap +SET RAM_DISK=R: && R:\.venv\Scripts\pipenv run pytest -n auto -v tests --tap-stream > report.tap diff --git a/run-tests.sh b/run-tests.sh index 493a90a8..b71fd4fb 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -4,9 +4,6 @@ set -eo pipefail -# Set the PYPI vendor URL for pytest-pypi. -PYPI_VENDOR_DIR="$(pwd)/tests/pypi/" -export PYPI_VENDOR_DIR export PYTHONIOENCODING="utf-8" export LANG=C.UTF-8 diff --git a/tasks/release.py b/tasks/release.py index c1946fa7..4a242ba5 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -41,7 +41,7 @@ def drop_dist_dirs(ctx): def build_dists(ctx): drop_dist_dirs(ctx) log('Building sdist using %s ....' % sys.executable) - for py_version in ['2.7', '3.6']: + for py_version in ['2.7', '3.6', '3.7']: env = {'PIPENV_PYTHON': py_version} ctx.run('pipenv install --dev', env=env) if py_version == '3.6': @@ -56,6 +56,12 @@ def upload_dists(ctx): ctx.run('twine upload dist/*') +@invoke.task +def generate_markdown(ctx): + log('Generating markdown from changelog...') + ctx.run('pandoc CHANGELOG.rst -f rst -t markdown -o CHANGELOG.md') + + @invoke.task def generate_changelog(ctx, commit=False, draft=False): log('Generating changelog...') @@ -91,16 +97,24 @@ def bump_version(ctx, dry_run=False, increment=True, release=False, dev=False, p if pre and not tag: print('Using "pre" requires a corresponding tag.') return - if release and not dev and not pre: + if release and not dev and not pre and increment: new_version = current_version.replace(release=today.timetuple()[:3]).clear(pre=True, dev=True) elif release and (dev or pre): - new_version = current_version.replace(release=today.timetuple()[:3]) + if increment: + new_version = current_version.replace(release=today.timetuple()[:3]) + else: + new_version = current_version if dev: new_version = new_version.bump_dev() elif pre: new_version = new_version.bump_pre(tag=tag) else: - new_version = current_version.replace(release=next_month) + if not release: + increment = False + if increment: + new_version = current_version.replace(release=next_month) + else: + new_version = current_version if dev: new_version = new_version.bump_dev() elif pre: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d71f0f89..8a8aebe6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -9,6 +9,7 @@ from pipenv.vendor import delegator from pipenv.vendor import requests from pipenv.vendor import six from pipenv.vendor import toml +from pytest_pypi.app import prepare_packages as prepare_pypi_packages if six.PY2: class ResourceWarning(Warning): @@ -30,6 +31,8 @@ def check_internet(): WE_HAVE_INTERNET = check_internet() TESTS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PYPI_VENDOR_DIR = os.path.join(TESTS_ROOT, 'pypi') +prepare_pypi_packages(PYPI_VENDOR_DIR) def pytest_runtest_setup(item): @@ -68,7 +71,6 @@ class _PipenvInstance(object): os.environ['PIPENV_DONT_USE_PYENV'] = '1' os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' os.environ['PIPENV_VENV_IN_PROJECT'] = '1' - os.environ['PYPI_VENDOR_DIR'] = os.path.join(TESTS_ROOT, 'pypi') if self.chdir: os.chdir(self.path) return self diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index e5a48127..ab6648f4 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -359,10 +359,16 @@ def test_install_venv_project_directory(PipenvInstance, pypi): os.environ["WORKON_HOME"] = workon_home.name if "PIPENV_VENV_IN_PROJECT" in os.environ: del os.environ["PIPENV_VENV_IN_PROJECT"] + c = p.pipenv("install six") assert c.return_code == 0 - project = Project() - assert Path(project.virtualenv_location).joinpath(".project").exists() + + venv_loc = None + for line in c.err.splitlines(): + if line.startswith("Virtualenv location:"): + venv_loc = Path(line.split(":", 1)[-1].strip()) + assert venv_loc is not None + assert venv_loc.joinpath(".project").exists() @pytest.mark.deploy diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 1ab34f1b..25195bfb 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -1,6 +1,5 @@ import pytest import os -import six from pipenv.utils import temp_environ @@ -348,6 +347,27 @@ requests = {git = "https://github.com/requests/requests.git", ref = "master", ed assert c.return_code == 0 +@pytest.mark.extras +@pytest.mark.lock +@pytest.mark.vcs +@pytest.mark.needs_internet +def test_lock_editable_vcs_with_extras_without_install(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[packages] +requests = {git = "https://github.com/requests/requests.git", editable = true, extras = ["socks"]} + """.strip()) + c = p.pipenv('lock') + assert c.return_code == 0 + assert 'requests' in p.lockfile['default'] + assert 'idna' in p.lockfile['default'] + assert 'chardet' in p.lockfile['default'] + assert "socks" in p.lockfile["default"]["requests"]["extras"] + c = p.pipenv('install') + assert c.return_code == 0 + + @pytest.mark.lock @pytest.mark.skip(reason="This doesn't work for some reason.") def test_lock_respecting_python_version(PipenvInstance, pypi): diff --git a/tests/pytest-pypi/pytest_pypi/app.py b/tests/pytest-pypi/pytest_pypi/app.py index 57a368b8..ba1a5437 100644 --- a/tests/pytest-pypi/pytest_pypi/app.py +++ b/tests/pytest-pypi/pytest_pypi/app.py @@ -4,9 +4,6 @@ import json import requests from flask import Flask, redirect, abort, render_template, send_file, jsonify -PYPI_VENDOR_DIR = os.environ.get('PYPI_VENDOR_DIR', './pypi') -PYPI_VENDOR_DIR = os.path.abspath(PYPI_VENDOR_DIR) - app = Flask(__name__) session = requests.Session() @@ -14,41 +11,47 @@ packages = {} class Package(object): - """docstring for Package""" + """Package represents a collection of releases from one or more directories""" def __init__(self, name): super(Package, self).__init__() self.name = name - self._releases = [] + self.releases = {} + self._package_dirs = set() @property - def releases(self): - r = [] - for release in self._releases: - release = release[len(PYPI_VENDOR_DIR):].replace('\\', '/') - r.append(release) - return r + def json(self): + for path in self._package_dirs: + try: + with open(os.path.join(path, 'api.json')) as f: + return json.load(f) + except FileNotFoundError: + pass def __repr__(self): return "/json') def json_for_package(package): try: - with open(os.path.sep.join([PYPI_VENDOR_DIR, package, 'api.json'])) as f: - return jsonify(json.load(f)) + return jsonify(packages[package].json) except Exception: pass r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) return jsonify(r.json()) + if __name__ == '__main__': + PYPI_VENDOR_DIR = os.environ.get('PYPI_VENDOR_DIR', './pypi') + PYPI_VENDOR_DIR = os.path.abspath(PYPI_VENDOR_DIR) + prepare_packages(PYPI_VENDOR_DIR) + app.run() diff --git a/tests/pytest-pypi/pytest_pypi/templates/package.html b/tests/pytest-pypi/pytest_pypi/templates/package.html index 36e70a88..26ba9eca 100644 --- a/tests/pytest-pypi/pytest_pypi/templates/package.html +++ b/tests/pytest-pypi/pytest_pypi/templates/package.html @@ -7,8 +7,8 @@

Links for {{ package.name }}

{% for release in package.releases %} - {{ release }} + {{ release }}
{% endfor %} - \ No newline at end of file +