don't skip Pipenv installation if there are git deps (#656)

* default to user python path for collectstatic runs

Signed-off-by: Kenneth Reitz <me@kennethreitz.org>

* bugfixes

Signed-off-by: Kenneth Reitz <me@kennethreitz.org>

* fix

Signed-off-by: Kenneth Reitz <me@kennethreitz.org>

CHANGELOG.md

@@ -2,24 +2,24 @@
 
 # 126
 
-Skip installs if Pipfile.lock hasn't changed, and uninstall stale dependencies with Pipenv.
-
-- No longer warn if there is no `Procfile`.
+Bugfixes.
 
 # 125
 
-Set `PYTHONPATH` during collectstatic runs, other updates.
-
-- Update Set `PYTHONPATH` during collectstatic runs.
-- Update Pipenv "3.6" to "3.6.4".
+Bugfixes.
 
 # 124
 
-Update buildpack to automatically install [dev-packages] (Pipenv) during Heroku CI builds.
+Update buildpack to automatically install `[dev-packages]` during Heroku CI Pipenv builds.
+
+- Skip installs if Pipfile.lock hasn't changed, and uninstall stale dependencies with Pipenv.
+- Set `PYTHONPATH` during collectstatic runs.
+- No longer warn if there is no `Procfile`.
+- Update Pipenv's "3.6" runtime specifier to point to "3.6.4".
 
 # 123
 
-Update gunicorn init.d script to allow overrides.
+Update gunicorn `init.d` script to allow overrides.
 
 # 122
 
@@ -31,7 +31,7 @@ Update default Python to v3.6.3.
 
 # 120
 
-Use Pipenv --deploy.
+Use `$ pipenv --deploy`.
 
 # 119
 

Makefile

@@ -5,7 +5,7 @@ test: test-heroku-16
 
 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings
-	@shellcheck -x bin/steps/collectstatic bin/steps/cryptography bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
+	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
 	@shellcheck -x bin/steps/hooks/*
 
 test-cedar-14:

bin/compile

@@ -160,7 +160,7 @@ mtime "python.install.time" "${start}"
 
 # Pipenv support.
 # shellcheck source=bin/steps/pipenv
-sub_env "$BIN_DIR/steps/pipenv"
+source "$BIN_DIR/steps/pipenv"
 
 # Uninstall removed dependencies with Pip.
 let start=$(nowms)
@@ -186,10 +186,6 @@ source "$BIN_DIR/steps/mercurial"
 # shellcheck source=bin/steps/pylibmc
 source "$BIN_DIR/steps/pylibmc"
 
-# Libffi support.
-# shellcheck source=bin/steps/cryptography
-source "$BIN_DIR/steps/cryptography"
-
 # Support for Geo libraries.
 # shellcheck source=bin/steps/geo-libs
 sub_env "$BIN_DIR/steps/geo-libs"

bin/steps/collectstatic

@@ -30,7 +30,8 @@ if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALL
     puts-step "$ python $MANAGE_FILE collectstatic --noinput"
 
     # Run collectstatic, cleanup some of the noisy output.
-    export PYTHONPATH=.
+    PYTHONPATH=${PYTHONPATH:-.}
+    export PYTHONPATH
     python "$MANAGE_FILE" collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent
     COLLECTSTATIC_STATUS="${PIPESTATUS[0]}"
 
@@ -39,6 +40,8 @@ if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALL
     # Display a warning if collectstatic failed.
     [ "$COLLECTSTATIC_STATUS" -ne 0 ] && {
 
+        mcount "failure.collectstatic"
+
         echo
         echo " !     Error while running '$ python $MANAGE_FILE collectstatic --noinput'."
         echo "       See traceback above for details."

bin/steps/cryptography

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-# This script serves as the cffi build step of the
-# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
-# compiler.
-#
-# A [buildpack](https://devcenter.heroku.com/articles/buildpacks) is an
-# adapter between a Python application and Heroku's runtime.
-#
-# This script is invoked by [`bin/compile`](/).
-
-if [[ "$STACK" == "heroku-16" ]]; then
-    # libffi is pre-installed in the stack image so there is no need to vendor it.
-    return 0
-fi
-
-# The location of the pre-compiled libffi binary.
-VENDORED_LIBFFI="${VENDOR_URL}/libraries/vendor/libffi.tar.gz"
-
-PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
-
-# Syntax sugar.
-# shellcheck source=bin/utils
-source "$BIN_DIR/utils"
-
-# If a package using cffi exists within requirements, use vendored libffi.
-if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[argon2] Django[argon2] django[bcrypt] Django[bcrypt] PyNaCl pyOpenSSL PyOpenSSL requests[security] misaka &> /dev/null) then
-
-  if [ ! -d ".heroku/vendor/lib/libffi-3.1" ]; then
-    echo "-----> Noticed cffi. Bootstrapping libffi."
-    mkdir -p .heroku/vendor
-    # Download and extract libffi into target vendor directory.
-    curl "$VENDORED_LIBFFI" -s | tar zxv -C .heroku/vendor &> /dev/null
-  fi
-
-  LIBFFI=$(pwd)/vendor
-  export LIBFFI
-fi

bin/steps/mercurial

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 
 # Install Mercurial if it appears to be required.
-if (grep -Fiq "hg+" requirements.txt) then
-    /app/.heroku/python/bin/pip install  mercurial | cleanup | indent
+if [[ -f "requirements.txt" ]]; then
+	if (grep -Fiq "hg+" requirements.txt) then
+		/app/.heroku/python/bin/pip install  mercurial | cleanup | indent
+	fi
 fi
+

bin/steps/pip-install

@@ -15,10 +15,19 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     fi
 
     set +e
-    
+
     # Measure that we're using pip.
     mcount "tool.pip"
 
+    # Count expected build failures.
+    if grep -q 'wsgiref' requirements.txt; then
+        mcount "failure.wsgiref"
+    fi
+    if grep -q '==0.0.0' requirements.txt; then
+        mcount "failure.none-version"
+    fi
+
+
     /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
     PIP_STATUS="${PIPESTATUS[0]}"
     set -e
@@ -35,8 +44,8 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
 
     echo
-    
-    # Install test dependencies, for CI. 
+
+    # Install test dependencies, for CI.
     if [ "$INSTALL_TEST" ]; then
         if [[ -f "$1/requirements-test.txt" ]]; then
             puts-step "Installing test dependencies…"

bin/steps/pip-uninstall

@@ -11,7 +11,10 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
 
     cp .heroku/python/requirements-declared.txt requirements-declared.txt
 
-    pip-diff --stale requirements-declared.txt requirements.txt --exclude setuptools pip wheel > .heroku/python/requirements-stale.txt
+
+    if ! pip-diff --stale requirements-declared.txt requirements.txt --exclude setuptools pip wheel > .heroku/python/requirements-stale.txt; then
+      mount "failure.bad-requirements"
+    fi
 
     rm -fr requirements-declared.txt
 

bin/steps/pipenv

@@ -4,73 +4,67 @@
 # export PIPENV_FORCE_COLOR=1
 # shellcheck source=bin/utils
 source $BIN_DIR/utils
-
+set -e
 
 if [[ -f Pipfile.lock ]]; then
     if [[ -f .heroku/python/Pipfile.lock.sha256 ]]; then
         if [[ $(openssl dgst -sha256 Pipfile.lock) == $(cat .heroku/python/Pipfile.lock.sha256) ]]; then
-            if [[ ! "$PIPENV_ALWAYS_INSTALL" ]]; then
-                echo "Skipping installation, as Pipfile.lock hasn't changed since last deploy." | indent
-                echo "To disable this functionality, run the following command:"
-                echo ""
-                echo "    $ heroku config:set PIPENV_ALWAYS_INSTALL=1" | indent
+            # Measure that we're using Pipenv.
+            mcount "tool.pipenv"
 
-                SKIP_PIPENV_INSTALL=1
+            # Don't skip installation of there are git deps.
+            if ! grep -q 'git' Pipfile.lock; then
+                echo "Skipping installation, as Pipfile.lock hasn't changed since last deploy." | indent
+
+                mcount "tool.pipenv"
+                export SKIP_PIPENV_INSTALL=1
+                export SKIP_PIP_INSTALL=1
             fi
+
         fi
     fi
 fi
 
 
 if [ ! "$SKIP_PIPENV_INSTALL" ]; then
-
     # Pipenv support (Generate requriements.txt with pipenv).
     if [[ -f Pipfile ]]; then
-        if [[ ! -f requirements.txt ]]; then
-            puts-step "Installing requirements with latest Pipenv…"
+        # Measure that we're using Pipenv.
+        mcount "tool.pipenv"
 
-            # Measure that we're using Pipenv.
-            mcount "tool.pipenv"
+        # Skip pip install, later.
+        export SKIP_PIP_INSTALL=1
 
-            # Set PIP_EXTRA_INDEX_URL
-            if [[ -r $ENV_DIR/PIP_EXTRA_INDEX_URL ]]; then
-                PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")"
-                export PIP_EXTRA_INDEX_URL
-            fi
+        # Set PIP_EXTRA_INDEX_URL
+        if [[ -r $ENV_DIR/PIP_EXTRA_INDEX_URL ]]; then
+            PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")"
+            export PIP_EXTRA_INDEX_URL
+        fi
 
-            # if [[ -f .heroku/python/requirements-declared.txt ]]; then
-            #     cp .heroku/python/requirements-declared.txt requirements.txt
-            # fi
+        # Install pipenv.
+        /app/.heroku/python/bin/pip install pipenv --upgrade &> /dev/null
 
-            # Install pipenv.
-            /app/.heroku/python/bin/pip install pipenv --upgrade &> /dev/null
+        # Install the dependencies.
+        if [[ ! -f Pipfile.lock ]]; then
+            puts-step "Installing dependencies with latest Pipenv…"
+            /app/.heroku/python/bin/pipenv install --system --skip-lock 2>&1 | indent
+        else
+            pipenv-to-pip Pipfile.lock > requirements.txt
+            "$BIN_DIR/steps/pip-uninstall"
+            cp requirements.txt .heroku/python/requirements-declared.txt
+            openssl dgst -sha256 Pipfile.lock > .heroku/python/Pipfile.lock.sha256
 
-            # Install the dependencies.
-            if [[ ! -f Pipfile.lock ]]; then
-                /app/.heroku/python/bin/pipenv install --system --skip-lock 2>&1 | indent
-            else
-                pipenv-to-pip Pipfile.lock > requirements.txt
-                "$BIN_DIR/steps/pip-uninstall"
-                cp requirements.txt .heroku/python/requirements-declared.txt
-                openssl dgst -sha256 Pipfile.lock > .heroku/python/Pipfile.lock.sha256
+            puts-step "Installing dependencies with latest Pipenv…"
+            /app/.heroku/python/bin/pipenv install --system --deploy 2>&1 | indent
+        fi
 
-                /app/.heroku/python/bin/pipenv install --system --deploy 2>&1 | indent
-            fi
-
-            # Install the test dependencies, for CI.
-            if [ "$INSTALL_TEST" ]; then
-                puts-step "Installing test dependencies…"
-                /app/.heroku/python/bin/pipenv install --dev --system --deploy 2>&1 | cleanup | indent
-            fi
-
-            # Skip pip install, later.
-            export SKIP_PIP_INSTALL=1
-
-            # Pip freeze, for compatibility.
-            pip freeze > requirements.txt
+        # Install the test dependencies, for CI.
+        if [ "$INSTALL_TEST" ]; then
+            puts-step "Installing test dependencies…"
+            /app/.heroku/python/bin/pipenv install --dev --system --deploy 2>&1 | cleanup | indent
         fi
     fi
 else
-    pipenv-to-pip Pipfile.lock > requirements.txt
     export SKIP_PIP_INSTALL=1
+    pipenv-to-pip Pipfile.lock > requirements.txt
 fi

vendor/pip-pop/pip-grep

@@ -8,6 +8,7 @@ Options:
   -h --help     Show this screen.
 """
 import os
+import sys
 from docopt import docopt
 from pip.req import parse_requirements
 from pip.index import PackageFinder
@@ -70,4 +71,8 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    try:
+        main()
+    except Exception:
+        sys.exit(1)
+

vendor/pipenv-to-pip

@@ -19,8 +19,5 @@ def main():
 
     print('\n'.join(packages))
 
-
-try:
+if __name__ == '__main__':
     main()
-except Exception:
-    pass