update the changelog (#470)

* Add a test for the cryptography (cffi) compile step

Since it's currently untested.

* Skip vendoring libffi/libmemcached on Heroku-16

Unlike for Cedar-14, Heroku-16 comes with these packages pre-installed:
https://github.com/heroku/stack-images/blob/5a341970cfc1f201014262ad64c3b3e47514f663/heroku-16/installed-packages.txt#L111
https://github.com/heroku/stack-images/blob/5a341970cfc1f201014262ad64c3b3e47514f663/heroku-16/installed-packages.txt#L172

As such the build scripts had already been made a no-op on Heroku-16:
https://github.com/heroku/heroku-buildpack-python/blob/fedae5ceda5a42f594012f911c3808dd5dc6fd9e/builds/libraries/vendor/libffi#L6-L9
https://github.com/heroku/heroku-buildpack-python/blob/fedae5ceda5a42f594012f911c3808dd5dc6fd9e/builds/libraries/vendor/libmemcache#L6-L9

...meaning the Heroku-16 archives for them on S3 contain zero files.

However until now, the buildpack was still unnecessarily downloading
and extracting these empty archives - and not just on the first compile
(like on cedar-14), but every compile since the directory check will
never succeed.

CHANGELOG.md

@@ -1,5 +1,28 @@
 # Python Buildpack Changelog
 
+# 116
+
+Vendoring improvements.
+
+- Geos libraries should work on Heroku-16 now.
+- The libffi/libmemcached vendoring step is now skipped on Heroku-16 (since they are installed in the base image).
+
+# 115
+
+Revert a pull request.
+
+- No longer using `sub_env` for `pip install` step.
+
+# 114
+
+- Bugfixes.
+
+Blacklisting `PYTHONHOME` and `PYTHONPATH` for older apps. Upgrades to nltk support.
+
+# 113
+
+Updates to Pipenv support.
+
 # 112
 
 Bugfix.
@@ -12,17 +35,17 @@ Linting, bugfixes.
 
 # 110
 
+Update default Python to 3.6.2.
+
+# 109
+
 Update Default Python to 3.6.1, bugfixes.
 
 - Fixed automatic pip uninstall of dependencies removed from requirements.txt.
 
-# 109
-
-Fix output for collectstatic step.
-
 # 108
 
-Updated setuptools.
+Fix output for collectstatic step.
 
 # 107
 

README.md

@@ -4,7 +4,7 @@
 
 [![Build Status](https://travis-ci.org/heroku/heroku-buildpack-python.svg?branch=master)](https://travis-ci.org/heroku/heroku-buildpack-python)
 
-This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/buildpacks) for Python apps, powered by [pip](https://pip.pypa.io/) and other excellent software.
+This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/buildpacks) for Python apps, powered by [Pipenv](http://docs.pipenv.org/en/latest/), [pip](https://pip.pypa.io/) and other excellent software.
 
 Recommended web frameworks include **Django** and **Flask**. The recommended webserver is **Gunicorn**. There are no restrictions around what software can be used (as long as it's pip-installable). Web processes must bind to `$PORT`, and only the HTTP protocol is permitted for incoming connections.
 
@@ -16,7 +16,7 @@ See it in Action
 Deploying a Python application couldn't be easier:
 
     $ ls
-    Procfile  requirements.txt  web.py
+    Pipfile		Procfile	web.py
 
     $ heroku create --buildpack heroku/python
 
@@ -24,16 +24,14 @@ Deploying a Python application couldn't be easier:
     ...
     -----> Python app detected
     -----> Installing python-3.6.2
-         $ pip install -r requirements.txt
+    -----> Installing pip
-           Collecting requests (from -r requirements.txt (line 1))
+    -----> Installing requirements with latest pipenv...
-             Downloading requests-2.12.4-py2.py3-none-any.whl (576KB)
+           ...
-           Installing collected packages: requests
+           Installing dependencies from Pipfile...
-           Successfully installed requests-2.12.4
-
     -----> Discovering process types
            Procfile declares types -> (none)
 
-A `requirements.txt` file must be present at the root of your application's repository.
+A `Pipfile` or `requirements.txt` must be present at the root of your application's repository.
 
 You can also specify the latest production release of this buildpack for upcoming builds of an existing application:
 

bin/steps/cryptography

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This script serves as the Cryptography build step of the
+# This script serves as the cffi build step of the
 # [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
 # compiler.
 #
@@ -9,6 +9,11 @@
 #
 # This script is invoked by [`bin/compile`](/).
 
+if [[ "$STACK" == "heroku-16" ]]; then
+    # libffi is pre-installed in the stack image so there is no need to vendor it.
+    return 0
+fi
+
 # The location of the pre-compiled libffi binary.
 VENDORED_LIBFFI="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/libffi.tar.gz"
 

bin/steps/geo-libs

@@ -30,6 +30,7 @@ if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
     curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
     curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null
     curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null
+    cp /usr/lib/x86_64-linux-gnu/libjasper.so* ".heroku/vendor/lib/."
   fi
 
   GDAL=$(pwd)/vendor

bin/steps/nltk

@@ -21,10 +21,10 @@ if sp-grep -s nltk; then
 
     if [ -f "$nltk_packages_definition" ]; then
 
-        nltk_packages=$(tr "\n" " " < "$nltk_packages_definition")
+        readarray -t nltk_packages < "$nltk_packages_definition"
-        puts-step "Downloading NLTK packages: $nltk_packages"
+        puts-step "Downloading NLTK packages: ${nltk_packages[*]}"
 
-        python -m nltk.downloader -d "$BUILD_DIR/.heroku/python/nltk_data" "$nltk_packages"  | indent
+        python -m nltk.downloader -d "$BUILD_DIR/.heroku/python/nltk_data" "${nltk_packages[@]}" | indent
         set_env NLTK_DATA "/app/.heroku/python/nltk_data"
 
     else
@@ -32,4 +32,3 @@ if sp-grep -s nltk; then
         puts-warn "Learn more: https://devcenter.heroku.com/articles/python-nltk"
     fi
 fi
-

bin/steps/pip-install

@@ -6,7 +6,8 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     puts-step "Installing requirements with pip"
 
     set +e
-    sub_env /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
+
+    /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
     PIP_STATUS="${PIPESTATUS[0]}"
     set -e
 

bin/steps/pipenv

@@ -9,7 +9,7 @@ if [[ -f Pipfile ]]; then
         /app/.heroku/python/bin/pip install pipenv --upgrade &> /dev/null
 
         # Install the dependencies.
-        /app/.heroku/python/bin/pipenv install --system 2>&1 | indent
+        /app/.heroku/python/bin/pipenv install --system --skip-lock 2>&1 | indent
 
         # Skip pip install, later.
         export SKIP_PIP_INSTALL=1

bin/steps/pipenv-python-version

@@ -2,7 +2,7 @@
 
 # Detect Python-version with Pipenv.
 
-if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
+if [[ -f $BUILD_DIR/Pipfile ]]; then
 
     if [[ ! -f $BUILD_DIR/runtime.txt ]]; then
         if [[ ! -f $BUILD_DIR/Pipfile.lock ]]; then

bin/steps/pylibmc

@@ -9,6 +9,11 @@
 #
 # This script is invoked by [`bin/compile`](/).
 
+if [[ "$STACK" == "heroku-16" ]]; then
+    # libmemcached is pre-installed in the stack image so there is no need to vendor it.
+    return 0
+fi
+
 # The location of the pre-compiled libmemcached binary.
 VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/libmemcache.tar.gz"
 
@@ -16,7 +21,6 @@ VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor
 # shellcheck source=bin/utils
 source "$BIN_DIR/utils"
 
-
 # If pylibmc exists within requirements, use vendored libmemcached.
 if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
 

test/fixtures/cffi/requirements.txt

@@ -0,0 +1 @@
+cffi

test/fixtures/nltk/nltk.txt

@@ -1 +1,2 @@
-wordnet
+city_database
+stopwords

test/run

@@ -24,7 +24,7 @@ testGEOS() {
 
 testNLTK() {
   compile "nltk"
-  assertCaptured "wordnet"
+  assertCaptured "Downloading NLTK packages: city_database stopwords"
   assertCapturedSuccess
 }
 
@@ -47,6 +47,12 @@ testPsycopg2() {
   assertCapturedSuccess
 }
 
+testCffi() {
+  compile "cffi"
+  assertCaptured "cffi"
+  assertCapturedSuccess
+}
+
 testPylibmc() {
   compile "pylibmc"
   assertCaptured "pylibmc"

test/utils

@@ -150,6 +150,11 @@ _assertContains()
     esac
 
     fail "${msg:-${default_msg}}"
+
+    if [ "${haystack_type}" == "file" ]; then
+      echo
+      cat "${haystack}"
+    fi
   fi
 }