Merge remote-tracking branch 'origin/master'

CHANGELOG.md

@@ -1,60 +1,23 @@
 # Python Buildpack Changelog
 
-# 118
-
-Improvements to Pipenv support.
-
-# 117
-
-Bug fix.
-
-# 116
-
-Vendoring improvements.
-
-- Geos libraries should work on Heroku-16 now.
-- The libffi/libmemcached vendoring step is now skipped on Heroku-16 (since they are installed in the base image).
-
-# 115
-
-Revert a pull request.
-
-- No longer using `sub_env` for `pip install` step.
-
-# 114
-
-- Bugfixes.
-
-Blacklisting `PYTHONHOME` and `PYTHONPATH` for older apps. Upgrades to nltk support.
-
-# 113
-
-Updates to Pipenv support.
-
-# 112
-
-Bugfix.
-
-- Fixed grep output bug.
-
 # 111
 
-Linting, bugfixes.
+Linting , bugfixes.
 
 # 110
 
-Update default Python to 3.6.2.
-
-# 109
-
 Update Default Python to 3.6.1, bugfixes.
 
 - Fixed automatic pip uninstall of dependencies removed from requirements.txt.
 
-# 108
+# 109
 
 Fix output for collectstatic step.
 
+# 108
+
+Updated setuptools.
+
 # 107
 
 Bugfix for C dependency installation.

README.md

@@ -4,7 +4,7 @@
 
 [![Build Status](https://travis-ci.org/heroku/heroku-buildpack-python.svg?branch=master)](https://travis-ci.org/heroku/heroku-buildpack-python)
 
-This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/buildpacks) for Python apps, powered by [Pipenv](http://docs.pipenv.org/en/latest/), [pip](https://pip.pypa.io/) and other excellent software.
+This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/buildpacks) for Python apps, powered by [pip](https://pip.pypa.io/) and other excellent software.
 
 Recommended web frameworks include **Django** and **Flask**. The recommended webserver is **Gunicorn**. There are no restrictions around what software can be used (as long as it's pip-installable). Web processes must bind to `$PORT`, and only the HTTP protocol is permitted for incoming connections.
 
@@ -16,7 +16,7 @@ See it in Action
 Deploying a Python application couldn't be easier:
 
     $ ls
-    Pipfile		Procfile	web.py
+    Procfile  requirements.txt  web.py
 
     $ heroku create --buildpack heroku/python
 
@@ -24,14 +24,16 @@ Deploying a Python application couldn't be easier:
     ...
     -----> Python app detected
     -----> Installing python-3.6.2
-    -----> Installing pip
+         $ pip install -r requirements.txt
-    -----> Installing requirements with latest pipenv...
+           Collecting requests (from -r requirements.txt (line 1))
-           ...
+             Downloading requests-2.12.4-py2.py3-none-any.whl (576KB)
-           Installing dependencies from Pipfile...
+           Installing collected packages: requests
+           Successfully installed requests-2.12.4
+
     -----> Discovering process types
            Procfile declares types -> (none)
 
-A `Pipfile` or `requirements.txt` must be present at the root of your application's repository.
+A `requirements.txt` file must be present at the root of your application's repository.
 
 You can also specify the latest production release of this buildpack for upcoming builds of an existing application:
 

bin/compile

@@ -182,7 +182,7 @@ source "$BIN_DIR/steps/cryptography"
 
 # Support for Geo libraries.
 # shellcheck source=bin/steps/geo-libs
-sub_env "$BIN_DIR/steps/geo-libs"
+sub-env "$BIN_DIR/steps/geo-libs"
 
 # GDAL support.
 # shellcheck source=bin/steps/gdal
@@ -202,7 +202,7 @@ mtime "pip.install.time" "${start}"
 
 # Support for NLTK corpora.
 let start=$(nowms)
-sub_env "$BIN_DIR/steps/nltk"
+sub-env "$BIN_DIR/steps/nltk"
 mtime "nltk.download.time" "${start}"
 
 # Support for pip install -e.
@@ -215,20 +215,20 @@ fi
 
 # Django collectstatic support.
 let start=$(nowms)
-sub_env "$BIN_DIR/steps/collectstatic"
+sub-env "$BIN_DIR/steps/collectstatic"
 mtime "collectstatic.time" "${start}"
 
 # Create .profile script for application runtime environment variables.
-set_env PATH "\$HOME/.heroku/python/bin:\$PATH"
+set-env PATH "\$HOME/.heroku/python/bin:\$PATH"
-set_env PYTHONUNBUFFERED true
+set-env PYTHONUNBUFFERED true
-set_env PYTHONHOME /app/.heroku/python
+set-env PYTHONHOME /app/.heroku/python
 
-set_env LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LIBRARY_PATH"
+set-env LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LIBRARY_PATH"
-set_env LD_LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LD_LIBRARY_PATH"
+set-env LD_LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LD_LIBRARY_PATH"
 
-set_default_env LANG en_US.UTF-8
+set-default-env LANG en_US.UTF-8
-set_default_env PYTHONHASHSEED random
+set-default-env PYTHONHASHSEED random
-set_default_env PYTHONPATH /app/
+set-default-env PYTHONPATH /app/
 
 # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.
 cp "$ROOT_DIR/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH"

bin/steps/cryptography

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This script serves as the cffi build step of the
+# This script serves as the Cryptography build step of the
 # [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
 # compiler.
 #
@@ -9,11 +9,6 @@
 #
 # This script is invoked by [`bin/compile`](/).
 
-if [[ "$STACK" == "heroku-16" ]]; then
-    # libffi is pre-installed in the stack image so there is no need to vendor it.
-    return 0
-fi
-
 # The location of the pre-compiled libffi binary.
 VENDORED_LIBFFI="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/libffi.tar.gz"
 

bin/steps/geo-libs

@@ -30,12 +30,6 @@ if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
     curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
     curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null
     curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null
-
-    # Copy libjasper from build image to slug.
-    if [[ "$STACK" == "heroku-16" ]]; then
-      cp /usr/lib/x86_64-linux-gnu/libjasper.so* ".heroku/vendor/lib/."
-    fi
-
   fi
 
   GDAL=$(pwd)/vendor

bin/steps/hooks/post_compile

@@ -3,5 +3,5 @@
 if [ -f bin/post_compile ]; then
     echo "-----> Running post-compile hook"
     chmod +x bin/post_compile
-    sub_env bin/post_compile
+    sub-env bin/post_compile
 fi

bin/steps/hooks/pre_compile

@@ -3,5 +3,5 @@
 if [ -f bin/pre_compile ]; then
     echo "-----> Running pre-compile hook"
     chmod +x bin/pre_compile
-    sub_env bin/pre_compile
+    sub-env bin/pre_compile
 fi

bin/steps/nltk

@@ -21,14 +21,15 @@ if sp-grep -s nltk; then
 
     if [ -f "$nltk_packages_definition" ]; then
 
-        readarray -t nltk_packages < "$nltk_packages_definition"
+        nltk_packages=$(tr "\n" " " < "$nltk_packages_definition")
-        puts-step "Downloading NLTK packages: ${nltk_packages[*]}"
+        puts-step "Downloading NLTK packages: $nltk_packages"
 
-        python -m nltk.downloader -d "$BUILD_DIR/.heroku/python/nltk_data" "${nltk_packages[@]}" | indent
+        python -m nltk.downloader -d "$BUILD_DIR/.heroku/python/nltk_data" "$nltk_packages"  | indent
-        set_env NLTK_DATA "/app/.heroku/python/nltk_data"
+        set-env NLTK_DATA "/app/.heroku/python/nltk_data"
 
     else
         puts-warn "'nltk.txt' not found, not downloading any corpora"
         puts-warn "Learn more: https://devcenter.heroku.com/articles/python-nltk"
     fi
 fi
+

bin/steps/pip-install

@@ -6,8 +6,7 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     puts-step "Installing requirements with pip"
 
     set +e
-
+    sub-env /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
-    /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
     PIP_STATUS="${PIPESTATUS[0]}"
     set -e
 

bin/steps/pipenv

@@ -3,18 +3,13 @@
 # Pipenv support (Generate requriements.txt with pipenv).
 if [[ -f Pipfile ]]; then
     if [[ ! -f requirements.txt ]]; then
-        puts-step "Installing requirements with latest Pipenv..."
+        puts-step "Installing requirements with latest pipenv..."
 
         # Install pipenv.
         /app/.heroku/python/bin/pip install pipenv --upgrade &> /dev/null
 
-        if [[ ! -f Pipfile.lock ]]; then
-            /app/.heroku/python/bin/pipenv install --system --skip-lock 2>&1 | indent
-        else
-            /app/.heroku/python/bin/pipenv install --system 2>&1 | indent
-        fi
         # Install the dependencies.
-
+        /app/.heroku/python/bin/pipenv install --system 2>&1 | indent
 
         # Skip pip install, later.
         export SKIP_PIP_INSTALL=1

bin/steps/pipenv-python-version

@@ -2,7 +2,7 @@
 
 # Detect Python-version with Pipenv.
 
-if [[ -f $BUILD_DIR/Pipfile ]]; then
+if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
 
     if [[ ! -f $BUILD_DIR/runtime.txt ]]; then
         if [[ ! -f $BUILD_DIR/Pipfile.lock ]]; then

bin/steps/pylibmc

@@ -9,11 +9,6 @@
 #
 # This script is invoked by [`bin/compile`](/).
 
-if [[ "$STACK" == "heroku-16" ]]; then
-    # libmemcached is pre-installed in the stack image so there is no need to vendor it.
-    return 0
-fi
-
 # The location of the pre-compiled libmemcached binary.
 VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/libmemcache.tar.gz"
 
@@ -21,6 +16,7 @@ VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor
 # shellcheck source=bin/utils
 source "$BIN_DIR/utils"
 
+
 # If pylibmc exists within requirements, use vendored libmemcached.
 if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
 

bin/utils

@@ -4,7 +4,7 @@ shopt -s nullglob
 
 # The standard library.
 if [[ ! -f  /tmp/stdlib.sh ]]; then
-  curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v8/stdlib.sh > /tmp/stdlib.sh
+  curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
 fi
 # shellcheck source=/dev/null
 source /tmp/stdlib.sh
@@ -53,6 +53,30 @@ deep-cp() {
 }
 
 
+sub-env() {
+
+  WHITELIST=${2:-''}
+  BLACKLIST=${3:-'^(GIT_DIR|STACK|PYTHONHOME|LD_LIBRARY_PATH|LIBRARY_PATH|PATH)$'}
+
+  # Python-specific variables.
+  export PYHONHOME=$BUILD_DIR/.heroku/python
+  export PYTHONPATH=$BUILD_DIR/
+
+  (
+    if [ -d "$ENV_DIR" ]; then
+      # shellcheck disable=SC2045
+      for e in $(ls "$ENV_DIR"); do
+        echo "$e" | grep -E "$WHITELIST" | grep -qvE "$BLACKLIST" &&
+        export "$e=$(cat "$ENV_DIR/$e")"
+        :
+      done
+    fi
+
+    "$@"
+
+  )
+}
+
 # Measure the size of the Python installation.
 measure-size() {
   echo "$(du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')"

builds/libraries/vendor/gdal

@@ -10,7 +10,7 @@ hash -r
 
 echo "Building gdal..."
 
-VERSION="2.2.1"
+VERSION="1.11.5"
 SOURCE_TARBALL="http://download.osgeo.org/gdal/${VERSION}/gdal-${VERSION}.tar.gz"
 
 curl -L $SOURCE_TARBALL | tar zx

builds/libraries/vendor/geos

@@ -10,16 +10,14 @@ hash -r
 
 echo "Building geos..."
 
-VERSION=3.6.2
+SOURCE_TARBALL='http://download.osgeo.org/geos/geos-3.4.3.tar.bz2'
-
-SOURCE_TARBALL="http://download.osgeo.org/geos/geos-${VERSION}.tar.bz2"
 
 curl -L $SOURCE_TARBALL | tar xj
 
-pushd "geos-${VERSION}"
+cd geos-3.4.3
 ./configure --prefix=$OUT_PREFIX --enable-static=no &&
 make
 make install
 
 # Cleanup
-popd
+cd ..

builds/libraries/vendor/proj

@@ -10,15 +10,14 @@ hash -r
 
 echo "Building gdal..."
 
-VERSION=4.9.3
+SOURCE_TARBALL='http://download.osgeo.org/proj/proj-4.8.0.tar.gz'
-SOURCE_TARBALL="http://download.osgeo.org/proj/proj-${VERSION}.tar.gz"
 
 curl -L $SOURCE_TARBALL | tar zx
 
-pushd "proj-${VERSION}"
+cd proj-4.8.0
-./configure --prefix=$OUT_PREFIX --enable-static=no
+./configure --prefix=$OUT_PREFIX --enable-static=no &&
 make
 make install
 
 # Cleanup
-popd
+cd ..

test/fixtures/cffi/requirements.txt

@@ -1 +0,0 @@
-cffi

test/fixtures/nltk/nltk.txt

@@ -1,2 +1 @@
-city_database
+wordnet
-stopwords

test/run

@@ -24,7 +24,7 @@ testGEOS() {
 
 testNLTK() {
   compile "nltk"
-  assertCaptured "Downloading NLTK packages: city_database stopwords"
+  assertCaptured "wordnet"
   assertCapturedSuccess
 }
 
@@ -47,12 +47,6 @@ testPsycopg2() {
   assertCapturedSuccess
 }
 
-testCffi() {
-  compile "cffi"
-  assertCaptured "cffi"
-  assertCapturedSuccess
-}
-
 testPylibmc() {
   compile "pylibmc"
   assertCaptured "pylibmc"

test/utils

@@ -150,11 +150,6 @@ _assertContains()
     esac
 
     fail "${msg:-${default_msg}}"
-
-    if [ "${haystack_type}" == "file" ]; then
-      echo
-      cat "${haystack}"
-    fi
   fi
 }