update the changelog (#470)

* Add a test for the cryptography (cffi) compile step

Since it's currently untested.

* Skip vendoring libffi/libmemcached on Heroku-16

Unlike for Cedar-14, Heroku-16 comes with these packages pre-installed:
https://github.com/heroku/stack-images/blob/5a341970cfc1f201014262ad64c3b3e47514f663/heroku-16/installed-packages.txt#L111
https://github.com/heroku/stack-images/blob/5a341970cfc1f201014262ad64c3b3e47514f663/heroku-16/installed-packages.txt#L172

As such the build scripts had already been made a no-op on Heroku-16:
https://github.com/heroku/heroku-buildpack-python/blob/fedae5ceda5a42f594012f911c3808dd5dc6fd9e/builds/libraries/vendor/libffi#L6-L9
https://github.com/heroku/heroku-buildpack-python/blob/fedae5ceda5a42f594012f911c3808dd5dc6fd9e/builds/libraries/vendor/libmemcache#L6-L9

...meaning the Heroku-16 archives for them on S3 contain zero files.

However until now, the buildpack was still unnecessarily downloading
and extracting these empty archives - and not just on the first compile
(like on cedar-14), but every compile since the directory check will
never succeed.

CHANGELOG.md

@@ -1,22 +1,51 @@
 # Python Buildpack Changelog
 
+# 116
+
+Vendoring improvements.
+
+- Geos libraries should work on Heroku-16 now.
+- The libffi/libmemcached vendoring step is now skipped on Heroku-16 (since they are installed in the base image).
+
+# 115
+
+Revert a pull request.
+
+- No longer using `sub_env` for `pip install` step.
+
+# 114
+
+- Bugfixes.
+
+Blacklisting `PYTHONHOME` and `PYTHONPATH` for older apps. Upgrades to nltk support.
+
+# 113
+
+Updates to Pipenv support.
+
+# 112
+
+Bugfix.
+
+- Fixed grep output bug.
+
 # 111
 
-Linting , bugfixes.
+Linting, bugfixes.
 
 # 110
 
+Update default Python to 3.6.2.
+
+# 109
+
 Update Default Python to 3.6.1, bugfixes.
 
 - Fixed automatic pip uninstall of dependencies removed from requirements.txt.
 
-# 109
-
-Fix output for collectstatic step.
-
 # 108
 
-Updated setuptools.
+Fix output for collectstatic step.
 
 # 107
 

README.md

@@ -4,7 +4,7 @@
 
 [![Build Status](https://travis-ci.org/heroku/heroku-buildpack-python.svg?branch=master)](https://travis-ci.org/heroku/heroku-buildpack-python)
 
-This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/buildpacks) for Python apps, powered by [pip](https://pip.pypa.io/) and other excellent software.
+This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/buildpacks) for Python apps, powered by [Pipenv](http://docs.pipenv.org/en/latest/), [pip](https://pip.pypa.io/) and other excellent software.
 
 Recommended web frameworks include **Django** and **Flask**. The recommended webserver is **Gunicorn**. There are no restrictions around what software can be used (as long as it's pip-installable). Web processes must bind to `$PORT`, and only the HTTP protocol is permitted for incoming connections.
 
@@ -16,7 +16,7 @@ See it in Action
 Deploying a Python application couldn't be easier:
 
     $ ls
-    Procfile  requirements.txt  web.py
+    Pipfile		Procfile	web.py
 
     $ heroku create --buildpack heroku/python
 
@@ -24,16 +24,14 @@ Deploying a Python application couldn't be easier:
     ...
     -----> Python app detected
     -----> Installing python-3.6.2
-         $ pip install -r requirements.txt
+    -----> Installing pip
-           Collecting requests (from -r requirements.txt (line 1))
+    -----> Installing requirements with latest pipenv...
-             Downloading requests-2.12.4-py2.py3-none-any.whl (576KB)
+           ...
-           Installing collected packages: requests
+           Installing dependencies from Pipfile...
-           Successfully installed requests-2.12.4
-
     -----> Discovering process types
            Procfile declares types -> (none)
 
-A `requirements.txt` file must be present at the root of your application's repository.
+A `Pipfile` or `requirements.txt` must be present at the root of your application's repository.
 
 You can also specify the latest production release of this buildpack for upcoming builds of an existing application:
 

bin/compile

@@ -182,7 +182,7 @@ source "$BIN_DIR/steps/cryptography"
 
 # Support for Geo libraries.
 # shellcheck source=bin/steps/geo-libs
-sub-env "$BIN_DIR/steps/geo-libs"
+sub_env "$BIN_DIR/steps/geo-libs"
 
 # GDAL support.
 # shellcheck source=bin/steps/gdal
@@ -202,7 +202,7 @@ mtime "pip.install.time" "${start}"
 
 # Support for NLTK corpora.
 let start=$(nowms)
-sub-env "$BIN_DIR/steps/nltk"
+sub_env "$BIN_DIR/steps/nltk"
 mtime "nltk.download.time" "${start}"
 
 # Support for pip install -e.
@@ -215,20 +215,20 @@ fi
 
 # Django collectstatic support.
 let start=$(nowms)
-sub-env "$BIN_DIR/steps/collectstatic"
+sub_env "$BIN_DIR/steps/collectstatic"
 mtime "collectstatic.time" "${start}"
 
 # Create .profile script for application runtime environment variables.
-set-env PATH "\$HOME/.heroku/python/bin:\$PATH"
+set_env PATH "\$HOME/.heroku/python/bin:\$PATH"
-set-env PYTHONUNBUFFERED true
+set_env PYTHONUNBUFFERED true
-set-env PYTHONHOME /app/.heroku/python
+set_env PYTHONHOME /app/.heroku/python
 
-set-env LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LIBRARY_PATH"
+set_env LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LIBRARY_PATH"
-set-env LD_LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LD_LIBRARY_PATH"
+set_env LD_LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LD_LIBRARY_PATH"
 
-set-default-env LANG en_US.UTF-8
+set_default_env LANG en_US.UTF-8
-set-default-env PYTHONHASHSEED random
+set_default_env PYTHONHASHSEED random
-set-default-env PYTHONPATH /app/
+set_default_env PYTHONPATH /app/
 
 # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.
 cp "$ROOT_DIR/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH"

bin/steps/cryptography

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This script serves as the Cryptography build step of the
+# This script serves as the cffi build step of the
 # [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
 # compiler.
 #
@@ -9,6 +9,11 @@
 #
 # This script is invoked by [`bin/compile`](/).
 
+if [[ "$STACK" == "heroku-16" ]]; then
+    # libffi is pre-installed in the stack image so there is no need to vendor it.
+    return 0
+fi
+
 # The location of the pre-compiled libffi binary.
 VENDORED_LIBFFI="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/libffi.tar.gz"
 

bin/steps/geo-libs

@@ -30,6 +30,7 @@ if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
     curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
     curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null
     curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null
+    cp /usr/lib/x86_64-linux-gnu/libjasper.so* ".heroku/vendor/lib/."
   fi
 
   GDAL=$(pwd)/vendor

bin/steps/hooks/post_compile

@@ -3,5 +3,5 @@
 if [ -f bin/post_compile ]; then
     echo "-----> Running post-compile hook"
     chmod +x bin/post_compile
-    sub-env bin/post_compile
+    sub_env bin/post_compile
 fi

bin/steps/hooks/pre_compile

@@ -3,5 +3,5 @@
 if [ -f bin/pre_compile ]; then
     echo "-----> Running pre-compile hook"
     chmod +x bin/pre_compile
-    sub-env bin/pre_compile
+    sub_env bin/pre_compile
 fi

bin/steps/nltk

@@ -21,15 +21,14 @@ if sp-grep -s nltk; then
 
     if [ -f "$nltk_packages_definition" ]; then
 
-        nltk_packages=$(tr "\n" " " < "$nltk_packages_definition")
+        readarray -t nltk_packages < "$nltk_packages_definition"
-        puts-step "Downloading NLTK packages: $nltk_packages"
+        puts-step "Downloading NLTK packages: ${nltk_packages[*]}"
 
-        python -m nltk.downloader -d "$BUILD_DIR/.heroku/python/nltk_data" "$nltk_packages"  | indent
+        python -m nltk.downloader -d "$BUILD_DIR/.heroku/python/nltk_data" "${nltk_packages[@]}" | indent
-        set-env NLTK_DATA "/app/.heroku/python/nltk_data"
+        set_env NLTK_DATA "/app/.heroku/python/nltk_data"
 
     else
         puts-warn "'nltk.txt' not found, not downloading any corpora"
         puts-warn "Learn more: https://devcenter.heroku.com/articles/python-nltk"
     fi
 fi
-

bin/steps/pip-install

@@ -6,7 +6,8 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     puts-step "Installing requirements with pip"
 
     set +e
-    sub-env /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
+
+    /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
     PIP_STATUS="${PIPESTATUS[0]}"
     set -e
 

bin/steps/pipenv

@@ -9,7 +9,7 @@ if [[ -f Pipfile ]]; then
         /app/.heroku/python/bin/pip install pipenv --upgrade &> /dev/null
 
         # Install the dependencies.
-        /app/.heroku/python/bin/pipenv install --system 2>&1 | indent
+        /app/.heroku/python/bin/pipenv install --system --skip-lock 2>&1 | indent
 
         # Skip pip install, later.
         export SKIP_PIP_INSTALL=1

bin/steps/pipenv-python-version

@@ -2,7 +2,7 @@
 
 # Detect Python-version with Pipenv.
 
-if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
+if [[ -f $BUILD_DIR/Pipfile ]]; then
 
     if [[ ! -f $BUILD_DIR/runtime.txt ]]; then
         if [[ ! -f $BUILD_DIR/Pipfile.lock ]]; then

bin/steps/pylibmc

@@ -9,6 +9,11 @@
 #
 # This script is invoked by [`bin/compile`](/).
 
+if [[ "$STACK" == "heroku-16" ]]; then
+    # libmemcached is pre-installed in the stack image so there is no need to vendor it.
+    return 0
+fi
+
 # The location of the pre-compiled libmemcached binary.
 VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/libmemcache.tar.gz"
 
@@ -16,7 +21,6 @@ VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor
 # shellcheck source=bin/utils
 source "$BIN_DIR/utils"
 
-
 # If pylibmc exists within requirements, use vendored libmemcached.
 if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
 

bin/utils

@@ -4,7 +4,7 @@ shopt -s nullglob
 
 # The standard library.
 if [[ ! -f  /tmp/stdlib.sh ]]; then
-  curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
+  curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v8/stdlib.sh > /tmp/stdlib.sh
 fi
 # shellcheck source=/dev/null
 source /tmp/stdlib.sh
@@ -53,30 +53,6 @@ deep-cp() {
 }
 
 
-sub-env() {
-
-  WHITELIST=${2:-''}
-  BLACKLIST=${3:-'^(GIT_DIR|STACK|PYTHONHOME|LD_LIBRARY_PATH|LIBRARY_PATH|PATH)$'}
-
-  # Python-specific variables.
-  export PYHONHOME=$BUILD_DIR/.heroku/python
-  export PYTHONPATH=$BUILD_DIR/
-
-  (
-    if [ -d "$ENV_DIR" ]; then
-      # shellcheck disable=SC2045
-      for e in $(ls "$ENV_DIR"); do
-        echo "$e" | grep -E "$WHITELIST" | grep -qvE "$BLACKLIST" &&
-        export "$e=$(cat "$ENV_DIR/$e")"
-        :
-      done
-    fi
-
-    "$@"
-
-  )
-}
-
 # Measure the size of the Python installation.
 measure-size() {
   echo "$(du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')"

test/fixtures/cffi/requirements.txt

@@ -0,0 +1 @@
+cffi

test/fixtures/nltk/nltk.txt

@@ -1 +1,2 @@
-wordnet
+city_database
+stopwords

test/run

@@ -24,7 +24,7 @@ testGEOS() {
 
 testNLTK() {
   compile "nltk"
-  assertCaptured "wordnet"
+  assertCaptured "Downloading NLTK packages: city_database stopwords"
   assertCapturedSuccess
 }
 
@@ -47,6 +47,12 @@ testPsycopg2() {
   assertCapturedSuccess
 }
 
+testCffi() {
+  compile "cffi"
+  assertCaptured "cffi"
+  assertCapturedSuccess
+}
+
 testPylibmc() {
   compile "pylibmc"
   assertCaptured "pylibmc"

test/utils

@@ -150,6 +150,11 @@ _assertContains()
     esac
 
     fail "${msg:-${default_msg}}"
+
+    if [ "${haystack_type}" == "file" ]; then
+      echo
+      cat "${haystack}"
+    fi
   fi
 }