Merge branch 'egg-links'

commit 554a8bbae6
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Dec 19 16:26:18 2016 -0500

    comment out debugging line

commit 6572ad3d44
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Dec 19 16:21:03 2016 -0500

    test

commit 35cabaeebc
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Dec 19 16:09:21 2016 -0500

    try .

commit 300285a92d
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Dec 19 16:05:22 2016 -0500

    find

commit 7a6f1eb010
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Dec 19 16:02:24 2016 -0500

    find

commit 66d754978e
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Dec 19 15:55:04 2016 -0500

    fix xargs

commit 9c222a9350
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Dec 19 15:53:04 2016 -0500

    copy src

commit 7f4273f47f
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Dec 19 15:45:20 2016 -0500

    new replace script

commit dd707c21e3
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Dec 19 15:40:35 2016 -0500

    re-write egg-links

CHANGELOG.md

@@ -1,5 +1,24 @@
 # Python Buildpack Changelog
 
+## v89
+
+Improved cache functionality and fix egg-links regression. 
+
+## v88
+
+Fixed bug with editable pip installations. 
+
+## v87
+
+Updated default Python 2.7.13.
+
+- Python 2.7.13 uses UCS-4 build, more compatibile with linux wheels.
+- Updated setuptools to v32.1.0.
+
+## v86
+
+Refactor and multi-buildpack compatibility.
+
 ## v85
 
 Packaging fix. 
@@ -27,8 +46,8 @@ Update to library detection mechnisms (pip-pop).
 
 Updated default Python to 2.7.11.
 
-- Updated setuptools to v8.1.2.
-- Updated pip to v23.1.0.
+- Updated pip to v8.1.2.
+- Updated setuptools to v23.1.0.
 
 ## v80 (2016-04-05)
 

Dockerfile

@@ -0,0 +1,11 @@
+FROM heroku/heroku:16-build
+
+WORKDIR /app
+ENV WORKSPACE_DIR=/app/builds
+
+RUN apt-get update && apt-get install -y python-pip
+
+# Install bob-builder application
+RUN pip install bob-builder==0.0.5
+
+COPY . /app

bin/compile

@@ -27,15 +27,14 @@ BUILD_DIR=$1
 CACHE_DIR=$2
 ENV_DIR=$3
 
-# Static configurations for virtualenv caches.
-VIRTUALENV_LOC=".heroku/venv"
-LEGACY_TRIGGER="lib/python2.7"
+# Export path environment variables for sub-scripts.
+export BIN_DIR ROOT_DIR BUILD_DIR CACHE_DIR ENV_DIR
 
-DEFAULT_PYTHON_VERSION="python-2.7.12"
+DEFAULT_PYTHON_VERSION="python-2.7.13"
 DEFAULT_PYTHON_STACK="cedar-14"
 PYTHON_EXE="/app/.heroku/python/bin/python"
 PIP_VERSION="9.0.1"
-SETUPTOOLS_VERSION="28.8.0"
+SETUPTOOLS_VERSION="32.1.0"
 
 # Common Problem Warnings
 export WARNINGS_LOG=$(mktemp)
@@ -54,7 +53,6 @@ export PATH=$PATH:$ROOT_DIR/vendor/pip-pop
 [ ! "$SLUG_ID" ] && SLUG_ID="defaultslug"
 [ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID
 [ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK
-[[ $BUILD_DIR == "/app" ]] && SKIP_MOVE_BUILD=1
 
 # Sanitizing environment variables.
 unset GIT_DIR PYTHONHOME PYTHONPATH
@@ -64,7 +62,6 @@ unset CYTOKINE_LOG_FILE GEM_PATH
 # Setup buildpack instrumentation.
 bpwatch init $LOGPLEX_KEY
 bpwatch build python $BUILDPACK_VERSION $REQUEST_ID
-TMP_APP_DIR=$CACHE_DIR/tmp_app_dir
 
 bpwatch start compile
 
@@ -74,33 +71,8 @@ source $BIN_DIR/utils
 # Import collection of warnings.
 source $BIN_DIR/warnings
 
-# Directory Hacks for path consistency.
-APP_DIR='/app'
-TMP_APP_DIR=$CACHE_DIR/tmp_app_dir
-
-# Skip these steps for Docker.
-if [[ ! "$SKIP_MOVE_BUILD" ]]; then
-
-  # Copy Anvil app dir to temporary storage...
-  bpwatch start anvil_appdir_stage
-  if [ "$SLUG_ID" ]; then
-    mkdir -p $TMP_APP_DIR
-    deep-mv $APP_DIR $TMP_APP_DIR
-  else
-    deep-rm $APP_DIR
-  fi
-  bpwatch stop anvil_appdir_stage
-
-  # Copy Application code in.
-  bpwatch start appdir_stage
-    deep-cp $BUILD_DIR $APP_DIR
-  bpwatch stop appdir_stage
-fi
-
-
-# Set new context.
-ORIG_BUILD_DIR=$BUILD_DIR
-BUILD_DIR=$APP_DIR
+# we need to put a bunch of symlinks in there later
+mkdir -p /app/.heroku
 
 # Set up outputs under new context
 PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
@@ -111,14 +83,14 @@ GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh"
 export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH
 
 # Prepend proper environment variables for Python use.
-export PATH=$BUILD_DIR/.heroku/python/bin:$BUILD_DIR/.heroku/vendor/bin:$PATH
+export PATH=/app/.heroku/python/bin:/app/.heroku/vendor/bin:$PATH
 export PYTHONUNBUFFERED=1
 export LANG=en_US.UTF-8
-export C_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
-export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
-export LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
-export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
-export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
+export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
+export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
+export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
+export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
+export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
 
 # Switch to the repo's context.
 cd $BUILD_DIR
@@ -160,31 +132,37 @@ fi
 # Prepare the cache.
 mkdir -p $CACHE_DIR
 
-# Purge "old-style" virtualenvs.
-bpwatch start clear_old_venvs
-  [ -d $CACHE_DIR/$LEGACY_TRIGGER ] && rm -fr $CACHE_DIR/.heroku/bin $CACHE_DIR/.heroku/lib $CACHE_DIR/.heroku/include
-  [ -d $CACHE_DIR/$VIRTUALENV_LOC ] && rm -fr $CACHE_DIR/.heroku/venv $CACHE_DIR/.heroku/src
-bpwatch stop clear_old_venvs
-
 # Restore old artifacts from the cache.
 bpwatch start restore_cache
-    mkdir -p .heroku
+  mkdir -p .heroku
+
+  cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
+  cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
+  cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
+  cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
+  cp -R $CACHE_DIR/.heroku/venv .heroku/ &> /dev/null || true
+
+  if [[ -d $CACHE_DIR/.heroku/src ]]; then
+    cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
+  fi
 
-    cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
-    cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
-    cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
-    cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
-    cp -R $CACHE_DIR/.heroku/venv .heroku/ &> /dev/null || true
 
 bpwatch stop restore_cache
 
-set +e
-# Create set-aside `.heroku` folder.
-mkdir .heroku &> /dev/null
-set -e
-
 mkdir -p $(dirname $PROFILE_PATH)
 
+# Make the directory for -e pip installations.
+mkdir -p /app/.heroku/src
+
+if [[ $BUILD_DIR != '/app' ]]; then
+  # python expects to reside in /app, so set up symlinks
+  # we will not remove these later so subsequent buildpacks can still invoke it
+  ln -s $BUILD_DIR/.heroku/python /app/.heroku/python
+  ln -s $BUILD_DIR/.heroku/vendor /app/.heroku/vendor
+  ln -s $BUILD_DIR/.heroku/venv /app/.heroku/venv
+  # NOTE: /app/.heroku/src also exists, but is copied manually later.
+fi
+
 # Install Python.
 source $BIN_DIR/steps/python
 
@@ -229,6 +207,11 @@ set-default-env PYTHONPATH /app/
 # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.
 cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH
 
+# Symlink the directory for -e pip installations
+if [[ $BUILD_DIR != '/app' ]]; then
+  mv /app/.heroku/src $BUILD_DIR/.heroku
+  ln -s /app/.heroku/src $BUILD_DIR/.heroku/src
+fi
 
 # Experimental post_compile hook.
 bpwatch start post_compile
@@ -243,6 +226,7 @@ bpwatch start dump_cache
   rm -rf $CACHE_DIR/.heroku/python-stack
   rm -rf $CACHE_DIR/.heroku/vendor
   rm -rf $CACHE_DIR/.heroku/venv
+  rm -rf $CACHE_DIR/.heroku/src
 
   mkdir -p $CACHE_DIR/.heroku
   cp -R .heroku/python $CACHE_DIR/.heroku/
@@ -250,23 +234,9 @@ bpwatch start dump_cache
   cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true
   cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true
   cp -R .heroku/venv $CACHE_DIR/.heroku/ &> /dev/null || true
+  cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true
 
 bpwatch stop dump_cache
 
 # Fin.
-if [[ ! "$SKIP_MOVE_BUILD" ]]; then
-
-  bpwatch start appdir_commit
-    deep-rm $ORIG_BUILD_DIR
-    deep-mv $BUILD_DIR $ORIG_BUILD_DIR
-  bpwatch stop appdir_commit
-
-  bpwatch start anvil_appdir_commit
-  if [ "$SLUG_ID" ]; then
-    deep-mv $TMP_APP_DIR $APP_DIR
-  fi
-
-  bpwatch stop anvil_appdir_commit
-  bpwatch stop compile
-fi
-
+bpwatch stop compile

bin/steps/pip-install

@@ -5,7 +5,7 @@ puts-cmd "pip install -r requirements.txt"
 [ "$FRESH_PYTHON" ] && bpwatch start pip_install_first
 
 set +e
-/app/.heroku/python/bin/pip install -r requirements.txt --exists-action=w --src=./.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
+/app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
 PIP_STATUS="${PIPESTATUS[0]}"
 set -e
 
@@ -20,6 +20,11 @@ fi
 cp requirements.txt .heroku/python/requirements-declared.txt
 /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
 
+# Replace egg-links with new paths for /app.
+# find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 cat
+find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#./app/#"
+find .heroku/python/lib/python*/site-packages/ -name "easy-install.pth" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#/app/#"
+
 [ ! "$FRESH_PYTHON" ] && bpwatch stop pip_install
 [ "$FRESH_PYTHON" ] && bpwatch stop pip_install_first
 

builds/libraries/autoconf

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/python/
-
-OUT_PREFIX=$1
-
-echo "Building autoconf..."
-
-
-SOURCE_TARBALL='https://ftp.gnu.org/gnu/autoconf/autoconf-2.68.tar.gz'
-curl -L $SOURCE_TARBALL | tar xz
-
-cd autoconf-2.68
-./configure --prefix=$OUT_PREFIX
-make
-make install

builds/libraries/vendor/libffi

@@ -3,6 +3,11 @@
 
 OUT_PREFIX=$1
 
+# Skip the build for heroku-16.
+if [[ $S3_PREFIX == "heroku-16" ]]; then
+    exit 0
+fi
+
 # Use new path, containing autoconf.
 export PATH="/app/.heroku/python/bin/:$PATH"
 hash -r

builds/libraries/vendor/libmemcache

@@ -3,6 +3,11 @@
 
 OUT_PREFIX=$1
 
+# Skip the build for heroku-16.
+if [[ $S3_PREFIX == "heroku-16" ]]; then
+    exit 0
+fi
+
 # fail hard
 set -o pipefail
 # fail harder

builds/runtimes/pypy3-5.5.0

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+
+OUT_PREFIX=$1
+
+echo "Building PyPy..."
+SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux64.tar.bz2'
+curl -L $SOURCE_TARBALL | tar jx
+cp -R pypy3-v5.5.0-linux64/* $OUT_PREFIX
+
+ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python

builds/runtimes/python-2.7.13

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python..."
+SOURCE_TARBALL='https://python.org/ftp/python/2.7.13/Python-2.7.13.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-2.7.13 src
+cd src
+
+./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
+make
+make install