updated changelog

This reverts commit 72710696a0.

CHANGELOG.md

@@ -1,5 +1,51 @@
 # Python Buildpack Changelog
 
+## v76 (2016-02-08)
+
+Improved Django collectstatic support.
+
+- `$ python manage.py collectstatic` will only be run if `Django` is present in `requirements.txt`.
+- If collectstatic fails, the build fails. Full traceback is provided.
+- `$DISABLE_COLLECTSTATIC`: skip collectstatic step completely (not new).
+- `$DEBUG_COLLECTSTATIC`: echo environment variables upon collectstatic failure.
+- Updated build output style.
+- New warning for outdated Python (via pip `InsecurePlatform` warning).
+
+## v75 (2016-01-29)
+
+Updated pip and Setuptools.
+
+## v74 (2015-12-29)
+
+Added warnings for lack of Procfile.
+
+## v72 (2015-12-07)
+
+Updated default Python to 2.7.11.
+
+## v72 (2015-12-03)
+
+Added friendly warnings for common build failures.
+
+## v70 (2015-10-29)
+
+Improved compatibility with multi and node.js buildpacks.
+
+## v69 (2015-10-12)
+
+Revert to v66.
+
+## v68 (2015-10-12)
+
+Fixed .heroku/venv error with modern apps.
+
+## v67 (2015-10-12)
+
+Further improved cache compatibility with multi and node.js buildpacks.
+
+## v66 (2015-10-09)
+
+Improved compatibility with multi and node.js buildpacks.
 
 ## v65 (2015-10-08)
 
@@ -38,4 +84,3 @@ Default Python is now latest 2.7.10. Updated Pip and Distribute.
 - Default Python version is v2.7.10
 - Setuptools updated to v16.0
 - Pip updated to v7.0.1
-

Readme.md

@@ -1,8 +1,10 @@
-# Heroku buildpack: Python
+# Heroku Buildpack: Python
 ![python-banner](https://cloud.githubusercontent.com/assets/51578/8914205/ecf2047c-346b-11e5-98c5-42547f9f4410.jpg)
 
 This is a [Heroku buildpack](http://devcenter.heroku.com/articles/buildpacks) for Python apps, powered by [pip](http://www.pip-installer.org/).
 
+This buildpack supports running Django and Flask apps.
+
 
 Usage
 -----
@@ -17,7 +19,7 @@ Example usage:
     $ git push heroku master
     ...
     -----> Python app detected
-    -----> Installing runtime (python-2.7.10)
+    -----> Installing runtime (python-2.7.11)
     -----> Installing dependencies using pip
            Downloading/unpacking requests (from -r requirements.txt (line 1))
            Installing collected packages: requests
@@ -40,13 +42,13 @@ Specify a Runtime
 You can also provide arbitrary releases Python with a `runtime.txt` file.
 
     $ cat runtime.txt
-    python-3.5.0
+    python-3.5.1
 
 Runtime options include:
 
-- python-2.7.10
-- python-3.5.0
-- pypy-2.6.1 (unsupported, experimental)
+- python-2.7.11
+- python-3.5.1
+- pypy-4.0.1 (unsupported, experimental)
 - pypy3-2.4.0 (unsupported, experimental)
 
 Other [unsupported runtimes](https://github.com/heroku/heroku-buildpack-python/tree/master/builds/runtimes) are available as well.

bin/compile

@@ -1,5 +1,13 @@
 #!/usr/bin/env bash
 
+# The Heroku Python Buildpack. This script accepts paramaters for a build
+# directory, a cache directory, and a directory for app environment variables.
+
+# Warning: there are a few hacks in this script to accomidate excellent builds
+# on Heroku. No gaurentee for external compatibility is made. However,
+# everything should work fine outside of the Heroku environment, if the
+# environment is setup correctly.
+
 # Usage:
 #
 #     $ bin/compile <build-dir> <cache-dir> <env-path>
@@ -19,20 +27,19 @@ BUILD_DIR=$1
 CACHE_DIR=$2
 ENV_DIR=$3
 
-
-CACHED_DIRS=".heroku"
-
 # Static configurations for virtualenv caches.
 VIRTUALENV_LOC=".heroku/venv"
 LEGACY_TRIGGER="lib/python2.7"
-PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
-WEBCONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/python.webconcurrency.sh"
 
-DEFAULT_PYTHON_VERSION="python-2.7.10"
+DEFAULT_PYTHON_VERSION="python-2.7.11"
 DEFAULT_PYTHON_STACK="cedar"
 PYTHON_EXE="/app/.heroku/python/bin/python"
-PIP_VERSION="7.1.2"
-SETUPTOOLS_VERSION="18.3.2"
+PIP_VERSION="8.0.2"
+SETUPTOOLS_VERSION="19.6"
+
+# Common Problem Warnings
+export WARNINGS_LOG=$(mktemp)
+export DEFAULT_PYTHON_VERSION
 
 # Setup bpwatch
 export PATH=$PATH:$ROOT_DIR/vendor/bpwatch
@@ -50,20 +57,22 @@ export PATH=$PATH:$ROOT_DIR/vendor/pip-pop
 
 # Sanitizing environment variables.
 unset GIT_DIR PYTHONHOME PYTHONPATH LD_LIBRARY_PATH LIBRARY_PATH
+unset RECEIVE_DATA SOURCE_VERSION RUN_KEY BUILD_INFO DEPLOY
+unset LOG_TOKEN DYNO CYTOKINE_LOG_FILE GEM_PATH
 
+# Setup buildpack instrumentation.
 bpwatch init $LOGPLEX_KEY
 bpwatch build python $BUILDPACK_VERSION $REQUEST_ID
 TMP_APP_DIR=$CACHE_DIR/tmp_app_dir
 
 bpwatch start compile
 
-
-# We'll need to send these statics to other scripts we `source`.
-export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH
-
 # Syntax sugar.
 source $BIN_DIR/utils
 
+# Import collection of warnings.
+source $BIN_DIR/warnings
+
 # Directory Hacks for path consistiency.
 APP_DIR='/app'
 TMP_APP_DIR=$CACHE_DIR/tmp_app_dir
@@ -83,7 +92,7 @@ if [[ ! "$DOCKER_BUILD" ]]; then
 
   # Copy Application code in.
   bpwatch start appdir_stage
-    deep-mv $BUILD_DIR $APP_DIR
+    deep-cp $BUILD_DIR $APP_DIR
   bpwatch stop appdir_stage
 fi
 
@@ -92,7 +101,14 @@ fi
 ORIG_BUILD_DIR=$BUILD_DIR
 BUILD_DIR=$APP_DIR
 
-# Prepend proper path buildpack use.
+# Set up outputs under new context
+PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
+WEBCONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/python.webconcurrency.sh"
+
+# We'll need to send these statics to other scripts we `source`.
+export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH
+
+# Prepend proper environment variables for Python use.
 export PATH=$BUILD_DIR/.heroku/python/bin:$BUILD_DIR/.heroku/vendor/bin:$PATH
 export PYTHONUNBUFFERED=1
 export LANG=en_US.UTF-8
@@ -105,12 +121,18 @@ export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/ven
 # Switch to the repo's context.
 cd $BUILD_DIR
 
+# Warn for lack of Procfile.
+if [[ ! -f Procfile ]]; then
+  puts-warn 'Warning: Your application is missing a Procfile. This file tells Heroku how to run your application.'
+  puts-warn 'Learn more: https://devcenter.heroku.com/articles/procfile'
+fi
+
 # Experimental pre_compile hook.
 bpwatch start pre_compile
   source $BIN_DIR/steps/hooks/pre_compile
 bpwatch stop pre_compile
 
-# If no requirements given, assume `setup.py develop`.
+# If no requirements.txt file given, assume `setup.py develop` is intended.
 if [ ! -f requirements.txt ]; then
   echo "-e ." > requirements.txt
 fi
@@ -133,7 +155,7 @@ if [ ! -f runtime.txt ]; then
   echo $DEFAULT_PYTHON_VERSION > runtime.txt
 fi
 
-# ### The Cache
+# Prepare the cache.
 mkdir -p $CACHE_DIR
 
 # Purge "old-style" virtualenvs.
@@ -144,9 +166,14 @@ bpwatch stop clear_old_venvs
 
 # Restore old artifacts from the cache.
 bpwatch start restore_cache
-  for dir in $CACHED_DIRS; do
-    cp -R $CACHE_DIR/$dir . &> /dev/null || true
-  done
+    mkdir -p .heroku
+
+    cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
+    cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
+    cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
+    cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
+    cp -R $CACHE_DIR/.heroku/venv .heroku/ &> /dev/null || true
+
 bpwatch stop restore_cache
 
 set +e
@@ -184,10 +211,7 @@ source $BIN_DIR/steps/pip-install
 sub-env $BIN_DIR/steps/collectstatic
 
 
-# ### Finalize
-#
-
-# Set context environment variables.
+# Create .profile script for application runtime environment variables.
 set-env PATH '$HOME/.heroku/python/bin:$PATH'
 set-env PYTHONUNBUFFERED true
 set-env PYTHONHOME /app/.heroku/python
@@ -208,16 +232,27 @@ bpwatch stop post_compile
 
 # Store new artifacts in cache.
 bpwatch start dump_cache
-  for dir in $CACHED_DIRS; do
-    rm -rf $CACHE_DIR/$dir
-    cp -R $dir $CACHE_DIR/
-  done
+
+  rm -rf $CACHE_DIR/.heroku/python
+  rm -rf $CACHE_DIR/.heroku/python-version
+  rm -rf $CACHE_DIR/.heroku/python-stack
+  rm -rf $CACHE_DIR/.heroku/vendor
+  rm -rf $CACHE_DIR/.heroku/venv
+
+  mkdir -p $CACHE_DIR/.heroku
+  cp -R .heroku/python $CACHE_DIR/.heroku/
+  cp -R .heroku/python-version $CACHE_DIR/.heroku/
+  cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true
+  cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true
+  cp -R .heroku/venv $CACHE_DIR/.heroku/ &> /dev/null || true
+
 bpwatch stop dump_cache
 
-# ### Fin.
+# Fin.
 if [[ ! "$DOCKER_BUILD" ]]; then
 
   bpwatch start appdir_commit
+    deep-rm $ORIG_BUILD_DIR
     deep-mv $BUILD_DIR $ORIG_BUILD_DIR
   bpwatch stop appdir_commit
 

bin/release

@@ -18,6 +18,6 @@ if [[ $MANAGE_FILE ]]; then
 cat <<EOF
 
 addons:
-  heroku-postgresql:hobby-dev
+  - heroku-postgresql
 EOF
 fi

bin/steps/collectstatic

@@ -1,36 +1,67 @@
 #!/usr/bin/env bash
 
+# Django Collectstatic runner. If you have Django installed, collectstatic will
+# automatically be executed as part of the build process. If collectstatic
+# fails, your build fails.
+
+# This functionality will only activate if Django is in requirements.txt.
+
+# Runtime arguments:
+#   - $DISABLE_COLLECTSTATIC: disables this functionality.
+#   - $DEBUG_COLLECTSTATIC: upon failure, print out environment variables.
+
 source $BIN_DIR/utils
 
+# Location of 'manage.py', if it exists.
 MANAGE_FILE=$(find . -maxdepth 3 -type f -name 'manage.py' -printf '%d\t%P\n' | sort -nk1 | cut -f2 | head -1)
 MANAGE_FILE=${MANAGE_FILE:-fakepath}
 
+# Legacy file-based support for $DISABLE_COLLECTSTATIC
 [ -f .heroku/collectstatic_disabled ] && DISABLE_COLLECTSTATIC=1
 
-bpwatch start collectstatic
+# Ensure that Django is explicitily specified in requirments.txt
+pip-grep -s requirements.txt django Django && DJANGO_INSTALLED=1
 
-if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ]; then
+bpwatch start collectstatic  # metrics collection
+
+if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then
     set +e
 
-    echo "-----> Preparing static assets"
-    # Check if collectstatic is configured properly.
-    python $MANAGE_FILE collectstatic --dry-run --noinput &> /dev/null && RUN_COLLECTSTATIC=true
+    puts-cmd "python $MANAGE_FILE collectstatic --noinput"
 
-    # Compile assets if collectstatic appears to be kosher.
-    if [ "$RUN_COLLECTSTATIC" ]; then
+    # Run collectstatic, cleanup some of the noisy output.
+    python $MANAGE_FILE collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent
+    COLLECTSTATIC_STATUS="${PIPESTATUS[0]}"
 
-        echo "       Running collectstatic..."
-        python $MANAGE_FILE collectstatic --noinput  2>&1 | sed '/^Copying/d;/^$/d;/^ /d' | indent
+    set -e
+
+    # Display a warning if collectstatic failed.
+    [ $COLLECTSTATIC_STATUS -ne 0 ] && {
+
+        echo
+        echo " !     Error while running '$ python $MANAGE_FILE collectstatic --noinput'."
+        echo "       See traceback above for details."
+        echo
+        echo "       You may need to update application code to resolve this error."
+        echo "       Or, you can disable collectstatic for this application:"
+        echo
+        echo "          $ heroku config:set DISABLE_COLLECTSTATIC=1"
+        echo
+        echo "       http://devcenter.heroku.com/articles/django-assets"
+
+        # Additionally, dump out the environment, if debug mode is on.
+        if [ "$DEBUG_COLLECTSTATIC" ]; then
+            echo
+            echo "****** Collectstatic environment variables:"
+            echo
+            env | indent
+        fi
+
+        # Abort the build.
+        exit 1
+    }
 
-        [ $? -ne 0 ] && {
-            echo " !     Error running 'manage.py collectstatic'. More info:"
-            echo "       http://devcenter.heroku.com/articles/django-assets"
-        }
-    else
-        echo "       Collectstatic configuration error. To debug, run:"
-        echo "       $ heroku run python $MANAGE_FILE collectstatic --noinput"
-    fi
     echo
 fi
 
-bpwatch stop collectstatic
+bpwatch stop collectstatic  # metrics collection

bin/steps/pip-install

@@ -1,16 +1,26 @@
 # Install dependencies with Pip.
-puts-step "Installing dependencies with pip"
+puts-cmd "pip install -r requirements.txt"
 
 [ ! "$FRESH_PYTHON" ] && bpwatch start pip_install
 [ "$FRESH_PYTHON" ] && bpwatch start pip_install_first
 
-/app/.heroku/python/bin/pip install -r requirements.txt --exists-action=w --src=./.heroku/src --allow-all-external --disable-pip-version-check --no-cache-dir | cleanup | indent
+set +e
+/app/.heroku/python/bin/pip install -r requirements.txt --exists-action=w --src=./.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
+PIP_STATUS="${PIPESTATUS[0]}"
+set -e
+
+show-warnings
+
+if [[ ! $PIP_STATUS -eq 0 ]]; then
+    exit 1
+fi
+
 
 # Smart Requirements handling
 cp requirements.txt .heroku/python/requirements-declared.txt
-/app/.heroku/python/bin/pip freeze > .heroku/python/requirements-installed.txt
+/app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
 
 [ ! "$FRESH_PYTHON" ] && bpwatch stop pip_install
 [ "$FRESH_PYTHON" ] && bpwatch stop pip_install_first
 
-echo
+echo

bin/steps/pip-uninstall

@@ -5,7 +5,7 @@ if [[ -f .heroku/python/requirements-declared.txt ]]; then
 
   cp .heroku/python/requirements-declared.txt requirements-declared.txt
 
-  pip-diff --stale requirements-declared.txt requirements.txt --exclude setuptools pip > .heroku/python/requirements-stale.txt
+  pip-diff --stale requirements-declared.txt requirements.txt --exclude setuptools pip wheel > .heroku/python/requirements-stale.txt
 
   rm -fr requirements-declared.txt
 

bin/steps/python

@@ -5,7 +5,7 @@ PYTHON_VERSION=$(cat runtime.txt)
 if [ -f .heroku/python-version ]; then
   if [ ! $(cat .heroku/python-version) = $PYTHON_VERSION ]; then
     bpwatch start uninstall_python
-      puts-step "Found runtime $(cat .heroku/python-version), removing"
+      puts-step "Found $(cat .heroku/python-version), removing"
       rm -fr .heroku/python
     bpwatch stop uninstall_python
   else
@@ -23,7 +23,7 @@ fi
 
 if [ ! "$SKIP_INSTALL" ]; then
   bpwatch start install_python
-    puts-step "Installing runtime ($PYTHON_VERSION)"
+    puts-step "Installing $PYTHON_VERSION"
 
     # Prepare destination directory.
     mkdir -p .heroku/python

bin/utils

@@ -16,6 +16,11 @@ cleanup() {
   sed -e 's/\.\.\.\+/.../g' | sed -e '/already satisfied/Id' | sed -e '/Overwriting/Id' | sed -e '/python executable/Id' | sed -e '/no previously-included files/Id'
 }
 
+# Buildpack Indented line.
+puts-line() {
+  echo "       $@"
+}
+
 # Buildpack Steps.
 puts-step() {
   echo "-----> $@"
@@ -26,6 +31,11 @@ puts-warn() {
   echo " !     $@"
 }
 
+# Buildpack Commands.
+puts-cmd() {
+  echo "     $ $@"
+}
+
 # Usage: $ set-env key value
 set-env() {
   echo "export $1=$2" >> $PROFILE_PATH
@@ -78,6 +88,10 @@ sub-env() {
   WHITELIST=${2:-''}
   BLACKLIST=${3:-'^(GIT_DIR|PYTHONHOME|LD_LIBRARY_PATH|LIBRARY_PATH|PATH)$'}
 
+  # Python-specific variables.
+  export PYHONHOME=$BUILD_DIR/.heroku/python
+  export PYTHONPATH=$BUILD_DIR/
+
   (
     if [ -d "$ENV_DIR" ]; then
       for e in $(ls $ENV_DIR); do
@@ -91,3 +105,4 @@ sub-env() {
 
   )
 }
+

bin/warnings

@@ -0,0 +1,49 @@
+shopt -s extglob
+
+old-platform() {
+  if grep -qi 'InsecurePlatformWarning' "$WARNINGS_LOG"; then
+    echo
+    puts-warn "Hello! It looks like your application is using an outdated version of Python."
+    puts-warn "This caused the security warning you saw above during the 'pip install' step."
+    puts-warn "We recommend '$DEFAULT_PYTHON_VERSION', which you can specify in a 'runtime.txt' file."
+    puts-warn "  -- Much Love, Heroku."
+  fi
+}
+
+pylibmc-missing() {
+  if grep -qi 'fatal error: libmemcached/memcached.h: No such file or directory' "$WARNINGS_LOG"; then
+    echo
+    puts-warn "Hello! There was a problem with your build related to libmemcache."
+    puts-warn "The Python library 'pylibmc' must be explicitly specified in 'requirements.txt' in order to build correctly."
+    puts-warn "Once you do that, everything should work as expected.   -- Much Love, Heroku."
+  fi
+}
+
+scipy-included() {
+  if grep -qi 'running setup.py install for scipy' "$WARNINGS_LOG"; then
+    echo
+    puts-warn "Hello! It looks like you're trying to use scipy on Heroku."
+    puts-warn "Unfortunately, at this time, we do not directly support this library."
+    puts-warn "There is, however, a buildpack available that makes it possible to use it on Heroku."
+    puts-warn "You can learn more here:  https://devcenter.heroku.com/articles/python-c-deps"
+    puts-warn "Sorry for the inconvenience.   -- Much Love, Heroku."
+  fi
+}
+
+distribute-included() {
+  if grep -qi 'Running setup.py install for distribute' "$WARNINGS_LOG"; then
+    echo
+    puts-warn "Hello! Your requirements.txt file contains the distribute package."
+    puts-warn "This library is automatically installed by Heroku and shouldn't be in"
+    puts-warn "Your requirements.txt file. This can cause unexpected behavior."
+    puts-warn "  -- Much Love, Heroku."
+  fi
+}
+
+show-warnings() {
+  old-platform
+  pylibmc-missing
+  scipy-included
+  distribute-included
+}
+

builds/runtimes/pypy-4.0.0

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+
+OUT_PREFIX=$1
+
+echo "Building PyPy..."
+SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy-4.0.0-linux64.tar.bz2'
+curl -L $SOURCE_TARBALL | tar jx
+cp -R pypy-4.0.0-linux64/* $OUT_PREFIX
+
+ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/pypy-4.0.1

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+
+OUT_PREFIX=$1
+
+echo "Building PyPy..."
+SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy-4.0.1-linux64.tar.bz2'
+curl -L $SOURCE_TARBALL | tar jx
+cp -R pypy-4.0.1-linux64/* $OUT_PREFIX
+
+ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/python-2.7.11

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python..."
+SOURCE_TARBALL='https://python.org/ftp/python/2.7.11/Python-2.7.11.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-2.7.11 src
+cd src
+
+./configure --prefix=$OUT_PREFIX  --with-ensurepip=no
+make
+make install

builds/runtimes/python-2.7.11rc1

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python..."
+SOURCE_TARBALL='https://www.python.org/ftp/python/2.7.11/Python-2.7.11rc1.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-2.7.11rc1 src
+cd src
+
+./configure --prefix=$OUT_PREFIX  --with-ensurepip=no
+make
+make install

builds/runtimes/python-3.5.1

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python..."
+SOURCE_TARBALL='https://python.org/ftp/python/3.5.1/Python-3.5.1.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.5.1 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no --enable-shared
+make
+make install
+
+ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python
+