Fix egg-links

commit cb2c57dcffe856ad547ad8fbd1907815713dc4a7
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:44:39 2017 -0500

    no files were found

commit 38f861f6c6dbb2825c6551f220e610bea619c27f
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:37:35 2017 -0500

    uninstall

commit db1db5d2a0ad364d646d378ccff62b9aa0257efd
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:36:41 2017 -0500

    pip cleanup

commit 2e16f233849f683ad9c9d00bad51c2dd5da11c18
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:35:13 2017 -0500

    uninstall

commit 078e43d2926b77a40f21026969ee930aa7ad0fee
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:31:39 2017 -0500

    fix

commit eff318eaf37161f0c496e130688b27d596b9cd7a
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:30:09 2017 -0500

    wfwef

commit d8955b452190b1b6a40049c94df564144c4607dc
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:28:30 2017 -0500

    oops

commit 0e83a0ae238bdea06ce0d184c8139b598f71745e
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:24:28 2017 -0500

    find

commit b9cb901ecb9d9075020f5c63e5faee04aade1ad7
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:23:04 2017 -0500

    don't

commit 3fd1a448e244a7c3f877ae75cd8672ef42a3f550
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:22:29 2017 -0500

    output

commit 884c6a40ee14365fccd4c9a34fb7733a833303a4
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:17:47 2017 -0500

    delete egg links

commit 9a16a8676abbd34b9ae0de3de4d52b4d358b2e35
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 11:14:18 2017 -0500

    remove egg-links

commit d45d9e977adc8ad7c5c18a27f1dfb20eb286bb5d
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 10:35:24 2017 -0500

    no echo

commit 5cfb64387db1c69fff57ac9afa0c996a34a4362d
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 10:33:58 2017 -0500

    debug

commit ac143097e9ee0a23464d16e2c6d414437046132e
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 10:26:27 2017 -0500

    no print 0

commit 72adfe2e4abd975f5df5350f06f93d1309ff4ed1
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 10:25:27 2017 -0500

    cat other one

commit e51e83ca3ec9a6710add90424f80f9bc7c3d5bf3
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 10:23:52 2017 -0500

    echo

commit def0231dd068fe4a854074bf42409ef373ac0977
Author: Kenneth Reitz <me@kennethreitz.org>
Date:   Mon Jan 2 10:20:31 2017 -0500

    cat the files

CHANGELOG.md

@@ -1,5 +1,52 @@
 # Python Buildpack Changelog
 
+## 96
+
+Bugfix.
+
+## 95
+
+Improved output support. 
+
+## v94
+
+Improved support for PyPy.
+
+## v93
+
+Improved support for PyPy.
+
+## v92
+
+Improved cache functionality and fix egg-links regression. 
+
+## v91
+
+Bugfix, rolled back to v88.
+
+## v90
+
+Bugfix.
+
+## v89
+
+Improved cache functionality and fix egg-links regression. 
+
+## v88
+
+Fixed bug with editable pip installations. 
+
+## v87
+
+Updated default Python 2.7.13.
+
+- Python 2.7.13 uses UCS-4 build, more compatibile with linux wheels.
+- Updated setuptools to v32.1.0.
+
+## v86
+
+Refactor and multi-buildpack compatibility.
+
 ## v85
 
 Packaging fix. 
@@ -27,8 +74,8 @@ Update to library detection mechnisms (pip-pop).
 
 Updated default Python to 2.7.11.
 
-- Updated setuptools to v8.1.2.
+- Updated pip to v8.1.2.
-- Updated pip to v23.1.0.
+- Updated setuptools to v23.1.0.
 
 ## v80 (2016-04-05)
 

Dockerfile

@@ -0,0 +1,11 @@
+FROM heroku/heroku:16-build
+
+WORKDIR /app
+ENV WORKSPACE_DIR=/app/builds
+
+RUN apt-get update && apt-get install -y python-pip
+
+# Install bob-builder application
+RUN pip install bob-builder==0.0.5
+
+COPY . /app

README.md

@@ -21,12 +21,12 @@ Deploying a Python application couldn't be easier:
     $ git push heroku master
     ...
     -----> Python app detected
-    -----> Installing python-2.7.12
+    -----> Installing python-2.7.13
          $ pip install -r requirements.txt
            Collecting requests (from -r requirements.txt (line 1))
-             Downloading requests-2.10.0-py2.py3-none-any.whl (501kB)
+             Downloading requests-2.12.4-py2.py3-none-any.whl (576KB)
            Installing collected packages: requests
-           Successfully installed requests-2.10.0
+           Successfully installed requests-2.12.4
            
     -----> Discovering process types
            Procfile declares types -> (none)
@@ -44,11 +44,11 @@ Specify a Python Runtime
 Specific versions of the Python runtime can be specified with a `runtime.txt` file:
 
     $ cat runtime.txt
-    python-3.5.2
+    python-3.6.0
 
 Runtime options include:
 
-- `python-2.7.12`
+- `python-2.7.13`
-- `python-3.5.2`
+- `python-3.6.0`
-- `pypy-5.3.1` (unsupported, experimental)
+- `pypy-5.6.0` (unsupported, experimental)
-- `pypy3-2.4.0` (unsupported, experimental)
+- `pypy3-5.5.0` (unsupported, experimental)

bin/compile

@@ -31,11 +31,11 @@ ENV_DIR=$3
 VIRTUALENV_LOC=".heroku/venv"
 LEGACY_TRIGGER="lib/python2.7"
 
-DEFAULT_PYTHON_VERSION="python-2.7.12"
+DEFAULT_PYTHON_VERSION="python-2.7.13"
 DEFAULT_PYTHON_STACK="cedar-14"
 PYTHON_EXE="/app/.heroku/python/bin/python"
 PIP_VERSION="9.0.1"
-SETUPTOOLS_VERSION="28.8.0"
+SETUPTOOLS_VERSION="32.1.0"
 
 # Common Problem Warnings
 export WARNINGS_LOG=$(mktemp)
@@ -54,7 +54,6 @@ export PATH=$PATH:$ROOT_DIR/vendor/pip-pop
 [ ! "$SLUG_ID" ] && SLUG_ID="defaultslug"
 [ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID
 [ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK
-[[ $BUILD_DIR == "/app" ]] && SKIP_MOVE_BUILD=1
 
 # Sanitizing environment variables.
 unset GIT_DIR PYTHONHOME PYTHONPATH
@@ -64,7 +63,6 @@ unset CYTOKINE_LOG_FILE GEM_PATH
 # Setup buildpack instrumentation.
 bpwatch init $LOGPLEX_KEY
 bpwatch build python $BUILDPACK_VERSION $REQUEST_ID
-TMP_APP_DIR=$CACHE_DIR/tmp_app_dir
 
 bpwatch start compile
 
@@ -74,33 +72,8 @@ source $BIN_DIR/utils
 # Import collection of warnings.
 source $BIN_DIR/warnings
 
-# Directory Hacks for path consistency.
+# we need to put a bunch of symlinks in there later
-APP_DIR='/app'
+mkdir -p /app/.heroku
-TMP_APP_DIR=$CACHE_DIR/tmp_app_dir
-
-# Skip these steps for Docker.
-if [[ ! "$SKIP_MOVE_BUILD" ]]; then
-
-  # Copy Anvil app dir to temporary storage...
-  bpwatch start anvil_appdir_stage
-  if [ "$SLUG_ID" ]; then
-    mkdir -p $TMP_APP_DIR
-    deep-mv $APP_DIR $TMP_APP_DIR
-  else
-    deep-rm $APP_DIR
-  fi
-  bpwatch stop anvil_appdir_stage
-
-  # Copy Application code in.
-  bpwatch start appdir_stage
-    deep-cp $BUILD_DIR $APP_DIR
-  bpwatch stop appdir_stage
-fi
-
-
-# Set new context.
-ORIG_BUILD_DIR=$BUILD_DIR
-BUILD_DIR=$APP_DIR
 
 # Set up outputs under new context
 PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
@@ -111,14 +84,14 @@ GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh"
 export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH
 
 # Prepend proper environment variables for Python use.
-export PATH=$BUILD_DIR/.heroku/python/bin:$BUILD_DIR/.heroku/vendor/bin:$PATH
+export PATH=/app/.heroku/python/bin:/app/.heroku/vendor/bin:$PATH
 export PYTHONUNBUFFERED=1
 export LANG=en_US.UTF-8
-export C_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
+export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
-export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:$BUILD_DIR/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
+export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
-export LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
+export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
-export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:$BUILD_DIR/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
-export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:$BUILD_DIR/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
+export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
 
 # Switch to the repo's context.
 cd $BUILD_DIR
@@ -175,15 +148,23 @@ bpwatch start restore_cache
     cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
     cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
     cp -R $CACHE_DIR/.heroku/venv .heroku/ &> /dev/null || true
+    if [[ -d $CACHE_DIR/.heroku/src ]]; then
+      cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
+    fi
 
 bpwatch stop restore_cache
 
-set +e
-# Create set-aside `.heroku` folder.
-mkdir .heroku &> /dev/null
-set -e
-
 mkdir -p $(dirname $PROFILE_PATH)
+mkdir -p /app/.heroku/src
+
+if [[ $BUILD_DIR != '/app' ]]; then
+    # python expects to reside in /app, so set up symlinks
+    # we will not remove these later so subsequent buildpacks can still invoke it
+    ln -nsf $BUILD_DIR/.heroku/python /app/.heroku/python
+    ln -nsf $BUILD_DIR/.heroku/vendor /app/.heroku/vendor
+    ln -nsf $BUILD_DIR/.heroku/venv /app/.heroku/venv
+    # Note: .heroku/src is copied in later.
+fi
 
 # Install Python.
 source $BIN_DIR/steps/python
@@ -209,13 +190,16 @@ sub-env $BIN_DIR/steps/geo-libs
 # GDAL support.
 source $BIN_DIR/steps/gdal
 
-# Install dependencies with Pip.
+# Install dependencies with Pip (where the magic happens).
 source $BIN_DIR/steps/pip-install
 
+# Support for pip install -e.
+rm -fr $BUILD_DIR/.heroku/src
+deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
+
 # Django collectstatic support.
 sub-env $BIN_DIR/steps/collectstatic
 
-
 # Create .profile script for application runtime environment variables.
 set-env PATH '$HOME/.heroku/python/bin:$PATH'
 set-env PYTHONUNBUFFERED true
@@ -229,12 +213,21 @@ set-default-env PYTHONPATH /app/
 # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.
 cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH
 
-
 # Experimental post_compile hook.
 bpwatch start post_compile
   source $BIN_DIR/steps/hooks/post_compile
 bpwatch stop post_compile
 
+set +e
+# rewrite build dir in egg links to /app so things are found at runtime
+find .heroku/python/lib/python*/site-packages/ -name "*.pth" -print0 2> /dev/null | xargs -r -0 -n 1 sed -i -e "s#$(pwd)#/app#" &> /dev/null
+set -e
+
+set +e
+# Support for PyPy
+find .heroku/python/lib-python/*/site-packages/  -name "*.pth" -print0 2> /dev/null | xargs -r -0 -n 1 sed -i -e "s#$(pwd)#/app#" &> /dev/null
+set -e
+
 # Store new artifacts in cache.
 bpwatch start dump_cache
 
@@ -243,6 +236,7 @@ bpwatch start dump_cache
   rm -rf $CACHE_DIR/.heroku/python-stack
   rm -rf $CACHE_DIR/.heroku/vendor
   rm -rf $CACHE_DIR/.heroku/venv
+  rm -rf $CACHE_DIR/.heroku/src
 
   mkdir -p $CACHE_DIR/.heroku
   cp -R .heroku/python $CACHE_DIR/.heroku/
@@ -250,23 +244,11 @@ bpwatch start dump_cache
   cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true
   cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true
   cp -R .heroku/venv $CACHE_DIR/.heroku/ &> /dev/null || true
+  if [[ -d .heroku/src ]]; then
+    cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true
+  fi
 
 bpwatch stop dump_cache
 
 # Fin.
-if [[ ! "$SKIP_MOVE_BUILD" ]]; then
+bpwatch stop compile
-
-  bpwatch start appdir_commit
-    deep-rm $ORIG_BUILD_DIR
-    deep-mv $BUILD_DIR $ORIG_BUILD_DIR
-  bpwatch stop appdir_commit
-
-  bpwatch start anvil_appdir_commit
-  if [ "$SLUG_ID" ]; then
-    deep-mv $TMP_APP_DIR $APP_DIR
-  fi
-
-  bpwatch stop anvil_appdir_commit
-  bpwatch stop compile
-fi
-

bin/steps/pip-install

@@ -1,11 +1,23 @@
 # Install dependencies with Pip.
 puts-cmd "pip install -r requirements.txt"
 
+set +e
+# delete any existing egg links, to uninstall exisisting installations.
+find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -delete 2> /dev/null
+find .heroku/python/lib/python*/site-packages/ -name "*.pth" -print0 2> /dev/null | xargs -r -0 -n 1 sed -i -e "s#/app/#/$(pwd)/#" &> /dev/null
+set -e
+
+set +e
+# Support for the above, for PyPy.
+find .heroku/python/lib-python/*/site-packages/ -name "*.egg-link" -print0 2> /dev/null | xargs -r -0 -n 1 sed -i -e "s#/app/#$(pwd)/#" &> /dev/null
+find .heroku/python/lib-python/*/site-packages/ -name "*.pth" -print0 2> /dev/null | xargs -r -0 -n 1 sed -i -e "s#/app/#/$(pwd)/#" &> /dev/null
+set -e
+
 [ ! "$FRESH_PYTHON" ] && bpwatch start pip_install
 [ "$FRESH_PYTHON" ] && bpwatch start pip_install_first
 
 set +e
-/app/.heroku/python/bin/pip install -r requirements.txt --exists-action=w --src=./.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
+/app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
 PIP_STATUS="${PIPESTATUS[0]}"
 set -e
 

bin/utils

@@ -13,7 +13,7 @@ indent() {
 
 # Clean up pip output
 cleanup() {
-  sed -e 's/\.\.\.\+/.../g' | sed -e '/already satisfied/Id' | sed -e '/Overwriting/Id' | sed -e '/python executable/Id' | sed -e '/no previously-included files/Id'
+  sed -e 's/\.\.\.\+/.../g' | sed -e '/already satisfied/Id' | sed -e '/No files were found to uninstall/Id' | sed -e '/Overwriting/Id' | sed -e '/python executable/Id' | sed -e '/no previously-included files/Id'
 }
 
 # Buildpack Indented line.

builds/libraries/autoconf

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/python/
-
-OUT_PREFIX=$1
-
-echo "Building autoconf..."
-
-
-SOURCE_TARBALL='https://ftp.gnu.org/gnu/autoconf/autoconf-2.68.tar.gz'
-curl -L $SOURCE_TARBALL | tar xz
-
-cd autoconf-2.68
-./configure --prefix=$OUT_PREFIX
-make
-make install

builds/libraries/vendor/libffi

@@ -3,6 +3,11 @@
 
 OUT_PREFIX=$1
 
+# Skip the build for heroku-16.
+if [[ $S3_PREFIX == "heroku-16" ]]; then
+    exit 0
+fi
+
 # Use new path, containing autoconf.
 export PATH="/app/.heroku/python/bin/:$PATH"
 hash -r

builds/libraries/vendor/libmemcache

@@ -3,6 +3,11 @@
 
 OUT_PREFIX=$1
 
+# Skip the build for heroku-16.
+if [[ $S3_PREFIX == "heroku-16" ]]; then
+    exit 0
+fi
+
 # fail hard
 set -o pipefail
 # fail harder

builds/runtimes/pypy3-5.5.0

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+
+OUT_PREFIX=$1
+
+echo "Building PyPy..."
+SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux64.tar.bz2'
+curl -L $SOURCE_TARBALL | tar jx
+cp -R pypy3-v5.5.0-linux64/* $OUT_PREFIX
+
+ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python

builds/runtimes/python-2.7.13

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python..."
+SOURCE_TARBALL='https://python.org/ftp/python/2.7.13/Python-2.7.13.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-2.7.13 src
+cd src
+
+./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
+make
+make install

builds/runtimes/python-3.6.0

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python..."
+SOURCE_TARBALL='https://python.org/ftp/python/3.6.0/Python-3.6.0.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.6.0 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python
+