diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index a2f34d4c..e39c45e7 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -8,6 +8,7 @@ trigger: exclude: - docs/* - news/* + - peeps/* - README.md - pipenv/*.txt - CHANGELOG.rst diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml index 35d5c16e..0f04afbe 100644 --- a/.azure-pipelines/windows.yml +++ b/.azure-pipelines/windows.yml @@ -8,6 +8,7 @@ trigger: exclude: - docs/* - news/* + - peeps/* - README.md - pipenv/*.txt - CHANGELOG.rst diff --git a/README.md b/README.md index 740f11d5..c3a46889 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ Pipenv: Python Development Workflow for Humans **Pipenv** is a tool that aims to bring the best of all packaging worlds (bundler, composer, npm, cargo, yarn, etc.) to the Python world. -*Windows is a first--class citizen, in our world.* +*Windows is a first-class citizen, in our world.* It automatically creates and manages a virtualenv for your projects, as well as adds/removes packages from your `Pipfile` as you -install/uninstall packages. It also generates the ever--important +install/uninstall packages. It also generates the ever-important `Pipfile.lock`, which is used to produce deterministic builds. ![image](http://media.kennethreitz.com.s3.amazonaws.com/pipenv.gif) diff --git a/docs/index.rst b/docs/index.rst index b6822409..a832ee8e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -49,7 +49,7 @@ You can quickly play with Pipenv right in your browser: Install Pipenv Today! --------------------- -If you're on MacOS, you can install Pipenv easily with Homebrew:: +If you're on MacOS, you can install Pipenv easily with Homebrew. You can also use Linuxbrew on Linux using the same command:: $ brew install pipenv diff --git a/docs/install.rst b/docs/install.rst index f529e249..4039cdb5 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -52,16 +52,16 @@ check this by running:: $ pip --version pip 9.0.1 -If you installed Python from source, with an installer from `python.org`_, or -via `Homebrew`_ you should already have pip. If you're on Linux and installed +If you installed Python from source, with an installer from `python.org`_, via `Homebrew`_ or via `Linuxbrew`_ you should already have pip. If you're on Linux and installed using your OS package manager, you may have to `install pip `_ separately. -If you plan to install Pipenv using Homebrew you can skip this step. The -Homebrew installer takes care of pip for you. +If you plan to install Pipenv using Homebrew or Linuxbrew you can skip this step. The +Homebrew/Linuxbrew installer takes care of pip for you. .. _getting started tutorial: https://opentechschool.github.io/python-beginners/en/getting_started.html#what-is-python-exactly .. _python.org: https://python.org .. _Homebrew: https://brew.sh +.. _Linuxbrew: https://linuxbrew.sh/ .. _Installing Python: http://docs.python-guide.org/en/latest/starting/installation/ @@ -83,13 +83,13 @@ cases. ☤ Homebrew Installation of Pipenv --------------------------------- -Homebrew is a popular open-source package management system for macOS. +`Homebrew`_ is a popular open-source package management system for macOS. For Linux users, `Linuxbrew`_ is a Linux port of that. -Installing pipenv via Homebrew will keep pipenv and all of its dependencies in +Installing pipenv via Homebrew or Linuxbrew will keep pipenv and all of its dependencies in an isolated virtual environment so it doesn't interfere with the rest of your Python installation. -Once you have installed `Homebrew`_ simply run:: +Once you have installed Homebrew or Linuxbrew simply run:: $ brew install pipenv diff --git a/news/2373.bugfix.rst b/news/2373.bugfix.rst new file mode 100644 index 00000000..9b42add1 --- /dev/null +++ b/news/2373.bugfix.rst @@ -0,0 +1 @@ +Raise `PipenvUsageError` when [[source]] does not contain url field. diff --git a/news/3339.bugfix b/news/3339.bugfix new file mode 100644 index 00000000..8e67e36f --- /dev/null +++ b/news/3339.bugfix @@ -0,0 +1 @@ +Fix a bug where custom virtualenv can not be activated with pipenv shell diff --git a/news/3348.feature.rst b/news/3348.feature.rst new file mode 100644 index 00000000..50547a3d --- /dev/null +++ b/news/3348.feature.rst @@ -0,0 +1 @@ +Added support for ``--verbose`` output to ``pipenv run``. \ No newline at end of file diff --git a/news/3351.bugfix.rst b/news/3351.bugfix.rst new file mode 100644 index 00000000..d2d9c675 --- /dev/null +++ b/news/3351.bugfix.rst @@ -0,0 +1 @@ +Fix a bug that ``--site-packages`` flag is not recognized. diff --git a/news/3362.trivial.rst b/news/3362.trivial.rst new file mode 100644 index 00000000..2216b071 --- /dev/null +++ b/news/3362.trivial.rst @@ -0,0 +1 @@ +The inline tables won't be rewritten now. diff --git a/news/3384.bugfix.rst b/news/3384.bugfix.rst new file mode 100644 index 00000000..f85cd168 --- /dev/null +++ b/news/3384.bugfix.rst @@ -0,0 +1 @@ +Fix unhashable type error during ``$ pipenv install --selective-upgrade`` diff --git a/news/3427.bugfix.rst b/news/3427.bugfix.rst new file mode 100644 index 00000000..42f659f0 --- /dev/null +++ b/news/3427.bugfix.rst @@ -0,0 +1 @@ +Fix a bug that ``ValidationError`` is thrown when some fields are missing in source section. diff --git a/news/3446.trivial.rst b/news/3446.trivial.rst new file mode 100644 index 00000000..1eb98bbb --- /dev/null +++ b/news/3446.trivial.rst @@ -0,0 +1 @@ +Fix the wrong order of old and new hashes in message. diff --git a/news/3449.bugfix.rst b/news/3449.bugfix.rst new file mode 100644 index 00000000..72e8d228 --- /dev/null +++ b/news/3449.bugfix.rst @@ -0,0 +1 @@ +Update the index names in lock file when source name in Pipfile is changed. diff --git a/peeps/PEEP-003.md b/peeps/PEEP-003.md new file mode 100644 index 00000000..d0493eaa --- /dev/null +++ b/peeps/PEEP-003.md @@ -0,0 +1,9 @@ +# PEEP-003: Revocation of Power of BDFL + +**ACCEPTED** + +Pipenv will be governed by a board of maintainers (trusted collaborators to the project on GitHub), not a BDFL. + +The BDFL retains his title, however, revokes himself of his powers. + +PEEP approval will be determined by available members of the board of maintainers, in private or public channels. diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index ae7111f1..ec1bef61 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -70,7 +70,6 @@ def cli( python=False, help=False, py=False, - site_packages=False, envs=False, man=False, completion=False, @@ -198,7 +197,7 @@ def cli( ) ctx.abort() # --two / --three was passed… - if (state.python or state.three is not None) or site_packages: + if (state.python or state.three is not None) or state.site_packages: ensure_project( three=state.three, python=state.python, diff --git a/pipenv/core.py b/pipenv/core.py index efe3cab0..8fae27c8 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1193,9 +1193,9 @@ def do_init( ) else: if old_hash: - msg = fix_utf8("Pipfile.lock ({1}) out of date, updating to ({0})…") + msg = fix_utf8("Pipfile.lock ({0}) out of date, updating to ({1})…") else: - msg = fix_utf8("Pipfile.lock is corrupted, replaced with ({0})…") + msg = fix_utf8("Pipfile.lock is corrupted, replaced with ({1})…") click.echo( crayons.red(msg.format(old_hash[-6:], new_hash[-6:]), bold=True), err=True, @@ -1836,7 +1836,7 @@ def do_install( if not is_star(section[package__name]) and is_star(package__val): # Support for VCS dependencies. package_args[i] = convert_deps_to_pip( - {packages: section[package__name]}, project=project, r=False + {package__name: section[package__name]}, project=project, r=False )[0] except KeyError: pass @@ -2146,11 +2146,6 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror three=three, python=python, validate=False, pypi_mirror=pypi_mirror, ) - # Set an environment variable, so we know we're in the environment. - os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") - - os.environ.pop("PIP_SHIMS_BASE_MODULE", None) - # Support shell compatibility mode. if PIPENV_SHELL_FANCY: fancy = True @@ -2166,6 +2161,13 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror shell_args, ) + # Set an environment variable, so we know we're in the environment. + # Only set PIPENV_ACTIVE after finishing reading virtualenv_location + # otherwise its value will be changed + os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") + + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + if fancy: shell.fork(*fork_args) return @@ -2300,16 +2302,24 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): three=three, python=python, validate=False, pypi_mirror=pypi_mirror, ) - # Set an environment variable, so we know we're in the environment. - os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") - - os.environ.pop("PIP_SHIMS_BASE_MODULE", None) load_dot_env() # Activate virtualenv under the current interpreter's environment inline_activate_virtual_environment() + + # Set an environment variable, so we know we're in the environment. + # Only set PIPENV_ACTIVE after finishing reading virtualenv_location + # such as in inline_activate_virtual_environment + # otherwise its value will be changed + os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") + + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + try: script = project.build_script(command, args) + cmd_string = ' '.join([script.command] + script.args) + if environments.is_verbose(): + click.echo(crayons.normal("$ {0}".format(cmd_string)), err=True) except ScriptEmptyError: click.echo("Can't run script {0!r}-it's empty?", err=True) if os.name == "nt": @@ -2495,7 +2505,7 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): if not project.virtualenv_exists: click.echo( u"{0}: No virtualenv has been created for this project yet! Consider " - u"running {1} first to automatically generate one for you or see" + u"running {1} first to automatically generate one for you or see " u"{2} for further instructions.".format( crayons.red("Warning", bold=True), crayons.green("`pipenv install`"), diff --git a/pipenv/project.py b/pipenv/project.py index e8550a04..edbcff87 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -780,7 +780,7 @@ class Project(object): return { "hash": {"sha256": self.calculate_pipfile_hash()}, "pipfile-spec": PIPFILE_SPEC_CURRENT, - "sources": sources, + "sources": [self.populate_source(s) for s in sources], "requires": self.parsed_pipfile.get("requires", {}) } diff --git a/pipenv/utils.py b/pipenv/utils.py index 9c64da79..5467300c 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -31,6 +31,7 @@ import crayons import parse from . import environments +from .exceptions import PipenvUsageError from .pep508checker import lookup @@ -83,19 +84,32 @@ def cleanup_toml(tml): def convert_toml_outline_tables(parsed): """Converts all outline tables to inline tables.""" - if isinstance(parsed, tomlkit.container.Container): - empty_inline_table = tomlkit.inline_table - else: - empty_inline_table = toml.TomlDecoder().get_empty_inline_table + def convert_tomlkit_table(section): + for key, value in section._body: + if not key: + continue + if hasattr(value, "keys") and not isinstance(value, tomlkit.items.InlineTable): + table = tomlkit.inline_table() + table.update(value.value) + section[key.key] = table + + def convert_toml_table(section): + for package, value in section.items(): + if hasattr(value, "keys") and not isinstance(value, toml.decoder.InlineTableDict): + table = toml.TomlDecoder().get_empty_inline_table() + table.update(value) + section[package] = table + + is_tomlkit_parsed = isinstance(parsed, tomlkit.container.Container) for section in ("packages", "dev-packages"): table_data = parsed.get(section, {}) - for package, value in table_data.items(): - if hasattr(value, "keys") and not isinstance( - value, (tomlkit.items.InlineTable, toml.decoder.InlineTableDict) - ): - table = empty_inline_table() - table.update(value) - table_data[package] = table + if not table_data: + continue + if is_tomlkit_parsed: + convert_tomlkit_table(table_data) + else: + convert_toml_table(table_data) + return parsed @@ -190,20 +204,26 @@ def prepare_pip_source_args(sources, pip_args=None): pip_args = [] if sources: # Add the source to notpip. - pip_args.extend(["-i", sources[0]["url"]]) + package_url = sources[0].get("url") + if not package_url: + raise PipenvUsageError("[[source]] section does not contain a URL.") + pip_args.extend(["-i", package_url]) # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urllib3_util.parse_url(sources[0]["url"]).host] + ["--trusted-host", urllib3_util.parse_url(package_url).host] ) # Add additional sources as extra indexes. if len(sources) > 1: for source in sources[1:]: - pip_args.extend(["--extra-index-url", source["url"]]) + url = source.get("url") + if not url: # not harmless, just don't continue + continue + pip_args.extend(["--extra-index-url", url]) # Trust the host if it's not verified. if not source.get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urllib3_util.parse_url(source["url"]).host] + ["--trusted-host", urllib3_util.parse_url(url).host] ) return pip_args @@ -221,9 +241,11 @@ def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): dep = " ".join(remainder) req = Requirement.from_line(dep) constraints.append(req.constraint_line) - if url: - index_lookup[req.name] = project.get_source(url=url).get("name") + source = first( + s for s in sources if s.get("url") and url.startswith(s["url"])) + if source: + index_lookup[req.name] = source.get("name") # strip the marker and re-add it later after resolution # but we will need a fallback in case resolution fails # eg pypiwin32 @@ -828,7 +850,7 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=True): dependencies = [] for dep_name, dep in deps.items(): - indexes = project.sources if hasattr(project, "sources") else [] + indexes = project.pipfile_sources if hasattr(project, "pipfile_sources") else [] new_dep = Requirement.from_pipfile(dep_name, dep) if new_dep.index: include_index = True diff --git a/setup.py b/setup.py index 4a6b9389..4c848c47 100644 --- a/setup.py +++ b/setup.py @@ -27,10 +27,7 @@ required = [ "setuptools>=36.2.1", "virtualenv-clone>=0.2.5", "virtualenv", - 'requests[security];python_version<"2.7"', - 'ordereddict;python_version<"2.7"', - 'enum34; python_version<"3"', - 'typing; python_version<"3.5"' + 'enum34; python_version<"3"' ] @@ -68,7 +65,7 @@ class DebCommand(Command): class UploadCommand(Command): - """Support setup.py publish.""" + """Support setup.py upload.""" description = "Build and publish the package." user_options = [] diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 91861db3..b5b5bc85 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -129,7 +129,7 @@ class _Pipfile(object): def __init__(self, path): self.path = path self.document = tomlkit.document() - self.document["sources"] = tomlkit.aot() + self.document["source"] = tomlkit.aot() self.document["requires"] = tomlkit.table() self.document["packages"] = tomlkit.table() self.document["dev_packages"] = tomlkit.table() @@ -155,7 +155,7 @@ class _Pipfile(object): source_table["url"] = os.environ.get("PIPENV_TEST_INDEX") source_table["verify_ssl"] = False source_table["name"] = "pipenv_test_index" - self.document["sources"].append(source_table) + self.document["source"].append(source_table) return tomlkit.dumps(self.document) def write(self): diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index 946b98f7..a38883e0 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -41,6 +41,20 @@ def test_pipenv_py(PipenvInstance): assert os.path.basename(python).startswith('python') +@pytest.mark.cli +def test_pipenv_site_packages(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('--python python --site-packages') + assert c.return_code == 0 + assert 'Making site-packages available' in c.err + + # no-global-site-packages.txt under stdlib dir should not exist. + c = p.pipenv('run python -c "import sysconfig; print(sysconfig.get_path(\'stdlib\'))"') + assert c.return_code == 0 + stdlib_path = c.out.strip() + assert not os.path.isfile(os.path.join(stdlib_path, 'no-global-site-packages.txt')) + + @pytest.mark.cli def test_pipenv_support(PipenvInstance): with PipenvInstance() as p: diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index 317351a0..f858f8ac 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -446,3 +446,25 @@ def test_install_package_with_dots(PipenvInstance, pypi): c = p.pipenv("install backports.html") assert c.ok assert "backports.html" in p.pipfile["packages"] + + +@pytest.mark.install +def test_rewrite_outline_table(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[packages] +six = {version = "*", editable = true} + +[packages.requests] +version = "*" + """.strip() + f.write(contents) + c = p.pipenv("install -e click") + assert c.return_code == 0 + with open(p.pipfile_path) as f: + contents = f.read() + assert "[packages.requests]" not in contents + assert 'six = {version = "*", editable = true}' in contents + assert 'requests = {version = "*"}' in contents + assert 'click = {' in contents diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index ab5178a0..d993b1a2 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -491,6 +491,7 @@ def test_lockfile_with_empty_dict(PipenvInstance): @pytest.mark.lock +@pytest.mark.skip_lock @pytest.mark.install def test_lock_with_incomplete_source(PipenvInstance, pypi): with PipenvInstance(pypi=pypi, chdir=True) as p: @@ -502,6 +503,8 @@ url = "https://test.pypi.org/simple" [packages] requests = "*" """) + c = p.pipenv('install --skip-lock') + assert c.return_code == 0 c = p.pipenv('install') assert c.return_code == 0 assert p.lockfile['_meta']['sources'] @@ -561,3 +564,27 @@ def test_vcs_lock_respects_top_level_pins(PipenvInstance, pypi): assert "git" in p.lockfile["default"]["requests"] assert "urllib3" in p.lockfile["default"] assert p.lockfile["default"]["urllib3"]["version"] == "==1.21.1" + + +@pytest.mark.lock +def test_lock_after_update_source_name(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + contents = """ +[[source]] +url = "https://test.pypi.org/simple" +verify_ssl = true +name = "test" + +[packages] +six = "*" + """.strip() + with open(p.pipfile_path, 'w') as f: + f.write(contents) + c = p.pipenv("lock") + assert c.return_code == 0 + assert p.lockfile["default"]["six"]["index"] == "test" + with open(p.pipfile_path, 'w') as f: + f.write(contents.replace('name = "test"', 'name = "custom"')) + c = p.pipenv("lock") + assert c.return_code == 0 + assert p.lockfile["default"]["six"]["index"] == "custom" diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index f9a2ba12..42754521 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -148,24 +148,6 @@ six = {{version = "*", index = "pypi"}} assert c.return_code == 0 -@pytest.mark.install -@pytest.mark.project -def test_rewrite_outline_table(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: - with open(p.pipfile_path, 'w') as f: - contents = """ -[packages.requests] -version = "*" - """.strip() - f.write(contents) - c = p.pipenv('install click') - assert c.return_code == 0 - with open(p.pipfile_path) as f: - contents = f.read() - assert "[packages.requests]" not in contents - assert 'requests = {version = "*"}' in contents - - @pytest.mark.install @pytest.mark.project def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpdir): @@ -185,9 +167,18 @@ def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpd @pytest.mark.project +@pytest.mark.virtualenv def test_run_in_virtualenv(PipenvInstance, pypi, virtualenv): with PipenvInstance(chdir=True, pypi=pypi) as p: + os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' + c = p.pipenv('run which pip') + assert c.return_code == 0 + assert 'virtualenv' not in c.out + os.environ.pop("PIPENV_IGNORE_VIRTUALENVS", None) + c = p.pipenv('run which pip') + assert c.return_code == 0 + assert 'virtualenv' in c.out project = Project() assert project.virtualenv_location == str(virtualenv) c = p.pipenv("run pip install click") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index aeaf3ed3..ac5eb29b 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8,6 +8,7 @@ from mock import Mock, patch import pipenv.utils import pythonfinder.utils +from pipenv.exceptions import PipenvUsageError # Pipfile format <-> requirements.txt format. @@ -374,6 +375,11 @@ twine = "*" == expected_args ) + def test_invalid_prepare_pip_source_args(self): + sources = [{}] + with pytest.raises(PipenvUsageError): + pipenv.utils.prepare_pip_source_args(sources, pip_args=None) + @pytest.mark.utils def test_parse_python_version(self): ver = pipenv.utils.parse_python_version("Python 3.6.5\n")