Merge remote-tracking branch 'origin/master'

* fixed the bug for pypy-5.8.0

* changelog entry

Signed-off-by: Kenneth Reitz <me@kennethreitz.org>

.gitignore

@@ -1,3 +1,5 @@
 *.pyc
 site
 .DS_Store
+
+/.envrc

.travis.yml

@@ -1,9 +1,23 @@
 language: bash
-sudo: required
-services:
-  - docker
-# install: docker pull heroku/cedar:14
-script: ./tests.sh
-env:
-  - STACK=heroku-16
-  - STACK=cedar-14
+dist: trusty
+jobs:
+  include:
+    - stage: "Bash linting (shellcheck)"
+      sudo: false
+      addons:
+        apt:
+          sources:
+          - debian-sid    # Grab shellcheck from the Debian repo (o_O)
+          packages:
+          - shellcheck
+      script: make check
+
+    - stage: "Stack Tests"
+      services: docker
+      env: STACK=heroku-16
+      script: ./tests.sh
+
+    - stage: "Stack Tests"
+      services: docker
+      env: STACK=cedar-14
+      script: ./tests.sh

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Python Buildpack Changelog
 
+# 111
+
+Linting , bugfixes.
+
 # 110
 
 Update Default Python to 3.6.1, bugfixes.

Makefile

@@ -1,7 +1,12 @@
 # These targets are not files
 .PHONY: tests
 
-test: test-cedar-14
+test: test-heroku-16
+
+check:
+	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings
+	@shellcheck -x bin/steps/collectstatic bin/steps/cryptography bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
+	@shellcheck -x bin/steps/hooks/*
 
 test-cedar-14:
 	@echo "Running tests in docker (cedar-14)..."

README.md

@@ -8,7 +8,7 @@ This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/bu
 
 Recommended web frameworks include **Django** and **Flask**. The recommended webserver is **Gunicorn**. There are no restrictions around what software can be used (as long as it's pip-installable). Web processes must bind to `$PORT`, and only the HTTP protocol is permitted for incoming connections.
 
-Some Python packages with obscure C dependencies (e.g. scipy) are [not compatible](https://devcenter.heroku.com/articles/python-c-deps).
+Some Python packages with obscure C dependencies are [not compatible](https://devcenter.heroku.com/articles/python-c-deps).
 
 See it in Action
 ----------------
@@ -23,7 +23,7 @@ Deploying a Python application couldn't be easier:
     $ git push heroku master
     ...
     -----> Python app detected
-    -----> Installing python-2.7.13
+    -----> Installing python-3.6.2
          $ pip install -r requirements.txt
            Collecting requests (from -r requirements.txt (line 1))
              Downloading requests-2.12.4-py2.py3-none-any.whl (576KB)
@@ -46,11 +46,11 @@ Specify a Python Runtime
 Specific versions of the Python runtime can be specified with a `runtime.txt` file:
 
     $ cat runtime.txt
-    python-3.6.1
+    python-2.7.13
 
 Runtime options include:
 
+- `python-3.6.2`
 - `python-2.7.13`
-- `python-3.6.1`
 - `pypy-5.7.1` (unsupported, experimental)
 - `pypy3-5.5.1` (unsupported, experimental)

bin/compile

@@ -25,20 +25,24 @@ export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
 export PATH=:/usr/local/bin:$PATH
 
 # Paths.
-BIN_DIR=$(cd $(dirname $0); pwd) # absolute path
-ROOT_DIR=$(dirname $BIN_DIR)
+BIN_DIR=$(cd "$(dirname "$0")"; pwd) # absolute path
+ROOT_DIR=$(dirname "$BIN_DIR")
 BUILD_DIR=$1
 CACHE_DIR=$2
 ENV_DIR=$3
 
+export BUILD_DIR CACHE_DIR ENV_DIR
+
 # Python defaults
-DEFAULT_PYTHON_VERSION="python-3.6.1"
+DEFAULT_PYTHON_VERSION="python-3.6.2"
 DEFAULT_PYTHON_STACK="cedar-14"
-PYTHON_EXE="/app/.heroku/python/bin/python"
-PIP_VERSION="9.0.1"
+PIP_UPDATE="9.0.1"
+
+export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE
 
 # Common Problem Warnings
-export WARNINGS_LOG=$(mktemp)
+WARNINGS_LOG=$(mktemp)
+export WARNINGS_LOG
 export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
 
 # Setup vendored tools and pip-pop (pip-diff)
@@ -55,10 +59,12 @@ unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN
 unset CYTOKINE_LOG_FILE GEM_PATH
 
 # Syntax sugar.
-source $BIN_DIR/utils
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
 
 # Import collection of warnings.
-source $BIN_DIR/warnings
+# shellcheck source=bin/warnings
+source "$BIN_DIR/warnings"
 
 # we need to put a bunch of symlinks in there later
 mkdir -p /app/.heroku
@@ -83,7 +89,7 @@ export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRA
 export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
 
 # Switch to the repo's context.
-cd $BUILD_DIR
+cd "$BUILD_DIR"
 
 # Warn for lack of Procfile.
 if [[ ! -f Procfile ]]; then
@@ -92,60 +98,66 @@ if [[ ! -f Procfile ]]; then
 fi
 
 # Prepare the cache.
-mkdir -p $CACHE_DIR
+mkdir -p "$CACHE_DIR"
 
 # Restore old artifacts from the cache.
 mkdir -p .heroku
 
-cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
-if [[ -d $CACHE_DIR/.heroku/src ]]; then
-  cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
+cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true
+cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
+cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
+cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true
+if [[ -d "$CACHE_DIR/.heroku/src" ]]; then
+  cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true
 fi
 
 # Experimental pre_compile hook.
-source $BIN_DIR/steps/hooks/pre_compile
+# shellcheck source=bin/steps/hooks/pre_compile
+source "$BIN_DIR/steps/hooks/pre_compile"
 
 # Sticky runtimes.
-if [ -f $CACHE_DIR/.heroku/python-version ]; then
-  DEFAULT_PYTHON_VERSION=$(cat $CACHE_DIR/.heroku/python-version)
+if [ -f "$CACHE_DIR/.heroku/python-version" ]; then
+  DEFAULT_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
 fi
 
 # Stack fallback for non-declared caches.
-if [ -f $CACHE_DIR/.heroku/python-stack ]; then
-  CACHED_PYTHON_STACK=$(cat $CACHE_DIR/.heroku/python-stack)
+if [ -f "$CACHE_DIR/.heroku/python-stack" ]; then
+  CACHED_PYTHON_STACK=$(cat "$CACHE_DIR/.heroku/python-stack")
 else
   CACHED_PYTHON_STACK=$STACK
 fi
 
+export CACHED_PYTHON_STACK
+
 # Pipenv Python version support.
-source $BIN_DIR/steps/pipenv-python-version
+# shellcheck source=bin/steps/pipenv-python-version
+source "$BIN_DIR/steps/pipenv-python-version"
 
 # If no runtime given, assume default version.
 if [ ! -f runtime.txt ]; then
-  echo $DEFAULT_PYTHON_VERSION > runtime.txt
+  echo "$DEFAULT_PYTHON_VERSION" > runtime.txt
 fi
 
-mkdir -p $(dirname $PROFILE_PATH)
+mkdir -p "$(dirname "$PROFILE_PATH")"
 mkdir -p /app/.heroku/src
 
 if [[ $BUILD_DIR != '/app' ]]; then
     # python expects to reside in /app, so set up symlinks
     # we will not remove these later so subsequent buildpacks can still invoke it
-    ln -nsf $BUILD_DIR/.heroku/python /app/.heroku/python
-    ln -nsf $BUILD_DIR/.heroku/vendor /app/.heroku/vendor
+    ln -nsf "$BUILD_DIR/.heroku/python" /app/.heroku/python
+    ln -nsf "$BUILD_DIR/.heroku/vendor" /app/.heroku/vendor
     # Note: .heroku/src is copied in later.
 fi
 
 # Install Python.
 let start=$(nowms)
-source $BIN_DIR/steps/python
+# shellcheck source=bin/steps/python
+source "$BIN_DIR/steps/python"
 mtime "python.install.time" "${start}"
 
 # Pipenv support.
-source $BIN_DIR/steps/pipenv
+# shellcheck source=bin/steps/pipenv
+source "$BIN_DIR/steps/pipenv"
 
 # If no requirements.txt file given, assume `setup.py develop` is intended.
 if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
@@ -153,88 +165,101 @@ if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
 fi
 
 # Fix egg-links.
-source $BIN_DIR/steps/eggpath-fix
+# shellcheck source=bin/steps/eggpath-fix
+source "$BIN_DIR/steps/eggpath-fix"
 
 # Mercurial support.
-source $BIN_DIR/steps/mercurial
+# shellcheck source=bin/steps/mercurial
+source "$BIN_DIR/steps/mercurial"
 
 # Pylibmc support.
-source $BIN_DIR/steps/pylibmc
+# shellcheck source=bin/steps/pylibmc
+source "$BIN_DIR/steps/pylibmc"
 
 # Libffi support.
-source $BIN_DIR/steps/cryptography
+# shellcheck source=bin/steps/cryptography
+source "$BIN_DIR/steps/cryptography"
 
 # Support for Geo libraries.
-sub-env $BIN_DIR/steps/geo-libs
+# shellcheck source=bin/steps/geo-libs
+sub-env "$BIN_DIR/steps/geo-libs"
 
 # GDAL support.
-source $BIN_DIR/steps/gdal
+# shellcheck source=bin/steps/gdal
+source "$BIN_DIR/steps/gdal"
 
 # Uninstall removed dependencies with Pip.
 let start=$(nowms)
-source $BIN_DIR/steps/pip-uninstall
+# shellcheck source=bin/steps/pip-uninstall
+source "$BIN_DIR/steps/pip-uninstall"
 mtime "pip.uninstall.time" "${start}"
 
 # Install dependencies with Pip (where the magic happens).
 let start=$(nowms)
-source $BIN_DIR/steps/pip-install
+# shellcheck source=bin/steps/pip-install
+source "$BIN_DIR/steps/pip-install"
 mtime "pip.install.time" "${start}"
 
 # Support for NLTK corpora.
 let start=$(nowms)
-sub-env $BIN_DIR/steps/nltk
+sub-env "$BIN_DIR/steps/nltk"
 mtime "nltk.download.time" "${start}"
 
 # Support for pip install -e.
 # In CI, $BUILD_DIR is /app.
 if [[ ! "$BUILD_DIR" == "/app" ]]; then
-  rm -fr $BUILD_DIR/.heroku/src
-  deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
+  rm -fr "$BUILD_DIR/.heroku/src"
+  deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src"
 fi
 
 
 # Django collectstatic support.
 let start=$(nowms)
-sub-env $BIN_DIR/steps/collectstatic
+sub-env "$BIN_DIR/steps/collectstatic"
 mtime "collectstatic.time" "${start}"
 
 # Create .profile script for application runtime environment variables.
-set-env PATH '$HOME/.heroku/python/bin:$PATH'
+set-env PATH "\$HOME/.heroku/python/bin:\$PATH"
 set-env PYTHONUNBUFFERED true
 set-env PYTHONHOME /app/.heroku/python
-set-env LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH'
-set-env LD_LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH'
+
+set-env LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LIBRARY_PATH"
+set-env LD_LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LD_LIBRARY_PATH"
+
 set-default-env LANG en_US.UTF-8
 set-default-env PYTHONHASHSEED random
 set-default-env PYTHONPATH /app/
 
 # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.
-cp $ROOT_DIR/vendor/WEB_CONCURRENCY.sh $WEB_CONCURRENCY_PROFILE_PATH
-cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH
+cp "$ROOT_DIR/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH"
+cp "$ROOT_DIR/vendor/python.gunicorn.sh" "$GUNICORN_PROFILE_PATH"
 
 
 # Experimental post_compile hook.
-source $BIN_DIR/steps/hooks/post_compile
+# shellcheck source=bin/steps/hooks/post_compile
+source "$BIN_DIR/steps/hooks/post_compile"
 
 # Fix egg-links, again.
-source $BIN_DIR/steps/eggpath-fix2
+# shellcheck source=bin/steps/eggpath-fix2
+source "$BIN_DIR/steps/eggpath-fix2"
 
 # Store new artifacts in cache.
 
-rm -rf $CACHE_DIR/.heroku/python
-rm -rf $CACHE_DIR/.heroku/python-version
-rm -rf $CACHE_DIR/.heroku/python-stack
-rm -rf $CACHE_DIR/.heroku/vendor
-rm -rf $CACHE_DIR/.heroku/src
+rm -rf "$CACHE_DIR/.heroku/python"
+rm -rf "$CACHE_DIR/.heroku/python-version"
+rm -rf "$CACHE_DIR/.heroku/python-stack"
+rm -rf "$CACHE_DIR/.heroku/vendor"
+rm -rf "$CACHE_DIR/.heroku/src"
 
-mkdir -p $CACHE_DIR/.heroku
-cp -R .heroku/python $CACHE_DIR/.heroku/
-cp -R .heroku/python-version $CACHE_DIR/.heroku/
-cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true
-cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true
+mkdir -p "$CACHE_DIR/.heroku"
+cp -R .heroku/python "$CACHE_DIR/.heroku/"
+cp -R .heroku/python-version "$CACHE_DIR/.heroku/"
+cp -R .heroku/python-stack "$CACHE_DIR/.heroku/" &> /dev/null || true
+cp -R .heroku/vendor "$CACHE_DIR/.heroku/" &> /dev/null || true
 if [[ -d .heroku/src ]]; then
-  cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true
+  cp -R .heroku/src "$CACHE_DIR/.heroku/" &> /dev/null || true
 fi
 
 # Measure the size of the Python installation.
+# shellcheck disable=SC2119
 mmeasure 'python.size' "$(measure-size)"

bin/detect

@@ -15,7 +15,7 @@
 BUILD_DIR=$1
 
 # Exit early if app is clearly not Python.
-if [ ! -f $BUILD_DIR/requirements.txt ] && [ ! -f $BUILD_DIR/setup.py ] && [ ! -f $BUILD_DIR/Pipfile ]; then
+if [ ! -f "$BUILD_DIR/requirements.txt" ] && [ ! -f "$BUILD_DIR/setup.py" ] && [ ! -f "$BUILD_DIR/Pipfile" ]; then
   exit 1
 fi
 

bin/release

@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
 # bin/release <build-dir>
 
-BIN_DIR=$(cd $(dirname $0); pwd) # absolute path
 BUILD_DIR=$1
 
-MANAGE_FILE=$(cd $BUILD_DIR && find . -maxdepth 3 -type f -name 'manage.py' | head -1)
+MANAGE_FILE=$(cd "$BUILD_DIR" && find . -maxdepth 3 -type f -name 'manage.py' | head -1)
 MANAGE_FILE=${MANAGE_FILE:2}
 
 cat <<EOF

bin/steps/collectstatic

@@ -10,6 +10,7 @@
 #   - $DISABLE_COLLECTSTATIC: disables this functionality.
 #   - $DEBUG_COLLECTSTATIC: upon failure, print out environment variables.
 
+# shellcheck source=bin/utils
 source $BIN_DIR/utils
 
 # Location of 'manage.py', if it exists.
@@ -29,13 +30,13 @@ if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALL
     puts-step "$ python $MANAGE_FILE collectstatic --noinput"
 
     # Run collectstatic, cleanup some of the noisy output.
-    python $MANAGE_FILE collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent
+    python "$MANAGE_FILE" collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent
     COLLECTSTATIC_STATUS="${PIPESTATUS[0]}"
 
     set -e
 
     # Display a warning if collectstatic failed.
-    [ $COLLECTSTATIC_STATUS -ne 0 ] && {
+    [ "$COLLECTSTATIC_STATUS" -ne 0 ] && {
 
         echo
         echo " !     Error while running '$ python $MANAGE_FILE collectstatic --noinput'."

bin/steps/cryptography

@@ -15,7 +15,8 @@ VENDORED_LIBFFI="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/li
 PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 
 # Syntax sugar.
-source $BIN_DIR/utils
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
 
 # If a package using cffi exists within requirements, use vendored libffi.
 if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[argon2] Django[argon2] django[bcrypt] Django[bcrypt] PyNaCl pyOpenSSL PyOpenSSL requests[security] misaka &> /dev/null) then
@@ -24,8 +25,9 @@ if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[arg
     echo "-----> Noticed cffi. Bootstrapping libffi."
     mkdir -p .heroku/vendor
     # Download and extract libffi into target vendor directory.
-    curl $VENDORED_LIBFFI -s | tar zxv -C .heroku/vendor &> /dev/null
+    curl "$VENDORED_LIBFFI" -s | tar zxv -C .heroku/vendor &> /dev/null
   fi
 
-  export LIBFFI=$(pwd)/vendor
+  LIBFFI=$(pwd)/vendor
+  export LIBFFI
 fi

bin/steps/eggpath-fix

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 set +e
 # delete any existing egg links, to uninstall exisisting installations.
 find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -delete 2> /dev/null

bin/steps/eggpath-fix2

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 set +e
 # rewrite build dir in egg links to /app so things are found at runtime
 find .heroku/python/lib/python*/site-packages/ -name "*.pth" -print0 2> /dev/null | xargs -r -0 -n 1 sed -i -e "s#$(pwd)#/app#" &> /dev/null

bin/steps/gdal

@@ -15,7 +15,8 @@ VENDORED_GDAL="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/gdal
 PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 
 # Syntax sugar.
-source $BIN_DIR/utils
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
 
 # If GDAL exists within requirements, use vendored gdal.
 if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
@@ -24,9 +25,10 @@ if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
     echo "-----> Noticed GDAL. Bootstrapping gdal."
     mkdir -p .heroku/vendor
     # Download and extract cryptography into target vendor directory.
-    curl $VENDORED_GDAL -s | tar zxv -C .heroku/vendor &> /dev/null
+    curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
   fi
 
-  export GDAL=$(pwd)/vendor
+  GDAL=$(pwd)/vendor
+  export GDAL
 fi
 

bin/steps/geo-libs

@@ -17,7 +17,8 @@ VENDORED_PROJ="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/proj
 PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 
 # Syntax sugar.
-source $BIN_DIR/utils
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
 
 # If GDAL exists within requirements, use vendored gdal.
 if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
@@ -26,11 +27,12 @@ if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
     echo "-----> Bootstrapping gdal, geos, proj."
     mkdir -p .heroku/vendor
     # Download and extract cryptography into target vendor directory.
-    curl $VENDORED_GDAL -s | tar zxv -C .heroku/vendor &> /dev/null
-    curl $VENDORED_GEOS -s | tar zxv -C .heroku/vendor &> /dev/null
-    curl $VENDORED_PROJ -s | tar zxv -C .heroku/vendor &> /dev/null
+    curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
+    curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null
+    curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null
   fi
 
-  export GDAL=$(pwd)/vendor
+  GDAL=$(pwd)/vendor
+  export GDAL
 fi
 

bin/steps/mercurial

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 # Install Mercurial if it appears to be required.
 if (grep -Fiq "hg+" requirements.txt) then
     /app/.heroku/python/bin/pip install  mercurial | cleanup | indent

bin/steps/nltk

@@ -10,18 +10,23 @@
 # This script is invoked by [`bin/compile`](/).
 
 # Syntax sugar.
-source $BIN_DIR/utils
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
 
 # Check that nltk was installed by pip, otherwise obviously not needed
-python -m nltk.downloader -h >/dev/null 2>&1
-if [ $? -eq 0 ]; then
+if sp-grep -s nltk; then
     puts-step "Downloading NLTK corpora..."
+
     nltk_packages_definition="$BUILD_DIR/nltk.txt"
+
     if [ -f "$nltk_packages_definition" ]; then
+
         nltk_packages=$(tr "\n" " " < "$nltk_packages_definition")
         puts-step "Downloading NLTK packages: $nltk_packages"
-        python -m nltk.downloader -d $BUILD_DIR/.heroku/python/nltk_data $nltk_packages  | indent
+
+        python -m nltk.downloader -d "$BUILD_DIR/.heroku/python/nltk_data" "$nltk_packages"  | indent
         set-env NLTK_DATA "/app/.heroku/python/nltk_data"
+
     else
         puts-warn "'nltk.txt' not found, not downloading any corpora"
         puts-warn "Learn more: https://devcenter.heroku.com/articles/python-nltk"

bin/steps/pip-install

@@ -1,10 +1,12 @@
+#!/usr/bin/env bash
+
 if [ ! "$SKIP_PIP_INSTALL" ]; then
 
     # Install dependencies with Pip.
     puts-step "Installing requirements with pip"
 
     set +e
-    /app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
+    sub-env /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
     PIP_STATUS="${PIPESTATUS[0]}"
     set -e
 
@@ -20,4 +22,4 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
 
     echo
-fi
+fi

bin/steps/pip-uninstall

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 set +e
 # Install dependencies with Pip.
 

bin/steps/pipenv

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 # Pipenv support (Generate requriements.txt with pipenv).
 if [[ -f Pipfile ]]; then
     if [[ ! -f requirements.txt ]]; then
@@ -16,4 +18,4 @@ if [[ -f Pipfile ]]; then
         /app/.heroku/python/bin/pip freeze > requirements.txt
 
     fi
-fi
+fi

bin/steps/pipenv-python-version

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 # Detect Python-version with Pipenv.
 
 if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
@@ -8,14 +10,14 @@ if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
         fi
         if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
             set +e
-            PYTHON=$(cat $BUILD_DIR/Pipfile.lock | jq '._meta.requires.python_version' -r)
+            PYTHON=$(jq -r '._meta.requires.python_version' "$BUILD_DIR/Pipfile.lock")
             set -e
 
             if [ "$PYTHON" = 2.7 ]; then
-                echo "python-2.7.13" > $BUILD_DIR/runtime.txt
+                echo "python-2.7.13" > "$BUILD_DIR/runtime.txt"
             fi
             if [ "$PYTHON" = 3.6 ]; then
-                echo "python-3.6.0" > $BUILD_DIR/runtime.txt
+                echo "python-3.6.2" > "$BUILD_DIR/runtime.txt"
             fi
         fi
     fi

bin/steps/pylibmc

@@ -13,7 +13,8 @@
 VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/libmemcache.tar.gz"
 
 # Syntax sugar.
-source $BIN_DIR/utils
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
 
 
 # If pylibmc exists within requirements, use vendored libmemcached.
@@ -23,8 +24,9 @@ if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
     echo "-----> Noticed pylibmc. Bootstrapping libmemcached."
     mkdir -p .heroku/vendor
     # Download and extract libmemcached into target vendor directory.
-    curl $VENDORED_MEMCACHED -s | tar zxv -C .heroku/vendor &> /dev/null
+    curl "$VENDORED_MEMCACHED" -s | tar zxv -C .heroku/vendor &> /dev/null
   fi
 
-  export LIBMEMCACHED=$(pwd)/vendor
+  LIBMEMCACHED=$(pwd)/vendor
+  export LIBMEMCACHED
 fi

bin/steps/python

@@ -1,10 +1,16 @@
+#!/usr/bin/env bash
+
 set +e
 runtime-fixer runtime.txt
 PYTHON_VERSION=$(cat runtime.txt)
 
-# Install Python.
+if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
+    puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache"
+    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
+fi
+
 if [ -f .heroku/python-version ]; then
-  if [ ! $(cat .heroku/python-version) = $PYTHON_VERSION ]; then
+  if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then
       puts-step "Found $(cat .heroku/python-version), removing"
       rm -fr .heroku/python
   else
@@ -12,41 +18,30 @@ if [ -f .heroku/python-version ]; then
   fi
 fi
 
-if [ ! $STACK = $CACHED_PYTHON_STACK ]; then
-    rm -fr .heroku/python .heroku/python-stack .heroku/vendor
-    unset SKIP_INSTALL
-fi
-
-
 if [ ! "$SKIP_INSTALL" ]; then
     puts-step "Installing $PYTHON_VERSION"
 
     # Prepare destination directory.
     mkdir -p .heroku/python
 
-    curl https://lang-python.s3.amazonaws.com/$STACK/runtimes/$PYTHON_VERSION.tar.gz -s | tar zxv -C .heroku/python &> /dev/null
     mcount "version.python.$PYTHON_VERSION"
 
-    if [[ $? != 0 ]] ; then
+    if ! curl "https://lang-python.s3.amazonaws.com/$STACK/runtimes/$PYTHON_VERSION.tar.gz" -s | tar zxv -C .heroku/python &> /dev/null; then
       puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)."
       puts-warn "Aborting.  More info: https://devcenter.heroku.com/articles/python-support"
       exit 1
     fi
 
   # Record for future reference.
-  echo $PYTHON_VERSION > .heroku/python-version
-  echo $STACK > .heroku/python-stack
+  echo "$PYTHON_VERSION" > .heroku/python-version
+  echo "$STACK" > .heroku/python-stack
   FRESH_PYTHON=true
 
   hash -r
 fi
 
 # If Pip isn't up to date:
-if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_VERSION* ]]; then
-  WORKING_DIR=$(pwd)
-
-  TMPTARDIR=$(mktemp -d)
-  trap "rm -rf $TMPTARDIR" RETURN
+if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
 
   puts-step "Installing pip"
 
@@ -54,7 +49,7 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_VERSION* ]]; then
   rm -fr /app/.heroku/python/lib/python2.7/site-packages/pip-*
   rm -fr /app/.heroku/python/lib/python2.7/site-packages/setuptools-*
 
-  /app/.heroku/python/bin/python $ROOT_DIR/vendor/get-pip.py &> /dev/null
+  /app/.heroku/python/bin/python "$ROOT_DIR/vendor/get-pip.py" &> /dev/null
 
 fi
 

bin/test-compile

@@ -1,10 +1,12 @@
 #!/usr/bin/env bash
 
 # Syntax sugar.
-BIN_DIR=$(cd $(dirname $0); pwd) # absolute path
-source $BIN_DIR/utils
+BIN_DIR=$(cd "$(dirname "$0")" || return; pwd) # absolute path
 
-DISABLE_COLLECTSTATIC=1 "$(dirname ${0:-})/compile" "$1" "$2" "$3"
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
+
+DISABLE_COLLECTSTATIC=1 "$(dirname "${0:-}")/compile" "$1" "$2" "$3"
 
 if [[ -f "$1/requirements-test.txt" ]]; then
     /app/.heroku/python/bin/pip install -r "$1/requirements-test.txt" --exists-action=w --src=./.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | cleanup | indent

bin/utils

@@ -1,13 +1,15 @@
 #!/usr/bin/env bash
 shopt -s extglob
+shopt -s nullglob
 
 # The standard library.
 if [[ ! -f  /tmp/stdlib.sh ]]; then
   curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
 fi
+# shellcheck source=/dev/null
 source /tmp/stdlib.sh
 
-if [ $(uname) == Darwin ]; then
+if [ "$(uname)" == Darwin ]; then
     sed() { command sed -l "$@"; }
 else
     sed() { command sed -u "$@"; }
@@ -26,12 +28,12 @@ cleanup() {
 
 # Buildpack Steps.
 puts-step() {
-  echo "-----> $@"
+  echo "-----> $*"
 }
 
 # Buildpack Warnings.
 puts-warn() {
-  echo " !     $@"
+  echo " !     $*"
 }
 
 # Does some serious copying.
@@ -62,20 +64,21 @@ sub-env() {
 
   (
     if [ -d "$ENV_DIR" ]; then
-      for e in $(ls $ENV_DIR); do
+      # shellcheck disable=SC2045
+      for e in $(ls "$ENV_DIR"); do
         echo "$e" | grep -E "$WHITELIST" | grep -qvE "$BLACKLIST" &&
-        export "$e=$(cat $ENV_DIR/$e)"
+        export "$e=$(cat "$ENV_DIR/$e")"
         :
       done
     fi
 
-    $1
+    "$@"
 
   )
 }
 
 # Measure the size of the Python installation.
 measure-size() {
-  echo "$((du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')"
+  echo "$(du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')"
 }
 

bin/warnings

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 shopt -s extglob
 
 old-platform() {

builds/libraries/vendor/gdal

@@ -10,14 +10,15 @@ hash -r
 
 echo "Building gdal..."
 
-SOURCE_TARBALL='http://download.osgeo.org/gdal/1.11.1/gdal-1.11.1.tar.gz'
+VERSION="1.11.5"
+SOURCE_TARBALL="http://download.osgeo.org/gdal/${VERSION}/gdal-${VERSION}.tar.gz"
 
 curl -L $SOURCE_TARBALL | tar zx
 
-cd gdal-1.11.1
+pushd "gdal-${VERSION}"
 ./configure --prefix=$OUT_PREFIX --enable-static=no  &&
 make
 make install
 
 # Cleanup
-cd ..
+popd

builds/runtimes/pypy-5.8.0

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building PyPy..."
+SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux64.tar.bz2'
+curl -L $SOURCE_TARBALL | tar jx
+cp -R pypy2-v5.8.0-linux64/* $OUT_PREFIX
+
+ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/python-2.7.13

@@ -13,3 +13,7 @@ cd src
 ./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
 make
 make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +

builds/runtimes/python-3.6.2

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python..."
+SOURCE_TARBALL='https://python.org/ftp/python/3.6.2/Python-3.6.2.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.6.2 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python

test/fixtures/geos/requirements.txt

@@ -0,0 +1 @@
+django

test/fixtures/python3/runtime.txt

@@ -1 +0,0 @@
-python-3.6.0

test/run

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 
-
 testPipenv() {
   compile "pipenv"
   assertCapturedSuccess
@@ -8,7 +7,7 @@ testPipenv() {
 
 testPipenvVersion() {
   compile "pipenv-version"
-  assertCaptured "3.6.0"
+  assertCaptured "3.6.2"
   assertCapturedSuccess
 }
 
@@ -17,6 +16,11 @@ testNoRequirements() {
   assertCapturedError
 }
 
+testGEOS() {
+  BUILD_WITH_GEO_LIBRARIES=1 compile "geos"
+  assertCaptured "geos"
+  assertCapturedSuccess
+}
 
 testNLTK() {
   compile "nltk"
@@ -24,7 +28,6 @@ testNLTK() {
   assertCapturedSuccess
 }
 
-
 testSetupPy() {
   compile "setup-py"
   assertCaptured "maya"
@@ -58,7 +61,7 @@ testPython2() {
 
 testPython3() {
   compile "python3"
-  assertCaptured "python-3.6.0"
+  assertCaptured "python-3.6.2"
   assertCapturedSuccess
 }
 
@@ -73,6 +76,15 @@ testSmartRequirements() {
   assertCapturedSuccess
 }
 
+testStackChange() {
+  local cache_dir="$(mktmpdir)"
+  mkdir -p "${cache_dir}/.heroku"
+  echo "different-stack" > "${cache_dir}/.heroku/python-stack"
+  compile "requirements-standard" "$cache_dir"
+  assertCaptured "clearing cache"
+  assertFile "$STACK" ".heroku/python-stack"
+  assertCapturedSuccess
+}
 
 
 pushd $(dirname 0) >/dev/null

vendor/WEB_CONCURRENCY.sh

@@ -5,25 +5,25 @@ case $(ulimit -u) in
 # Standard-1X (+Free, +Hobby) Dyno
 256)
   export DYNO_RAM=512
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-02}
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-2}
   ;;
 
 # Standard-2X Dyno
 512)
   export DYNO_RAM=1024
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-04}
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-4}
   ;;
 
 # Performance-M Dyno
 16384)
   export DYNO_RAM=2560
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-08}
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-8}
   ;;
 
 # Performance-L Dyno
 32768)
-  export DYNO_RAM=6656
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-011}
+  export DYNO_RAM=14336
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-11}
   ;;
 
-esac
+esac