Merge pull request #722 from heroku/python-3.7-and-3.6.6

Build Python 3.6.6 and 3.7.0

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @heroku/languages

.travis.yml

@@ -4,12 +4,9 @@ jobs:
   include:
     - stage: "Bash linting (shellcheck)"
       sudo: false
-      addons:
-        apt:
-          sources:
-          - debian-sid    # Grab shellcheck from the Debian repo (o_O)
-          packages:
-          - shellcheck
+      before_install:
+        - wget -c https://goo.gl/ZzKHFv -O - | tar -xvJ -C /tmp/
+        - PATH="/tmp/shellcheck-latest:$PATH"
       script: make check
 
     - stage: "Stack Tests"

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Python Buildpack Changelog
 
+# 136
+
+Upgrade to 3.6.6 and support 3.7.0 on all runtimes.
+
+# 135
+
+Upgrade Pipenv to v2018.5.18.
+
+# 134
+
+Default to 3.6.5, bugfixes.
+
+# 133
+
+Fixes for Pip 10 release.
+
+# 132
+
+Improve pip installation, with the release of v9.0.2.
+
 # 131
 
 Fix bug with pip.

README.md

@@ -23,9 +23,9 @@ Deploying a Python application couldn't be easier:
     $ git push heroku master
     …
     -----> Python app detected
-    -----> Installing python-3.6.4
+    -----> Installing python-3.6.6
     -----> Installing pip
-    -----> Installing requirements with Pipenv 11.7.1…
+    -----> Installing requirements with Pipenv 2018.5.18…
            ...
            Installing dependencies from Pipfile…
     -----> Discovering process types
@@ -49,14 +49,15 @@ Specific versions of the Python runtime can be specified in your `Pipfile`:
 Or, more specifically:
 
     [requires]
-    python_full_version = "2.7.14"
+    python_full_version = "2.7.15"
 
 Or, with a `runtime.txt` file:
 
     $ cat runtime.txt
-    python-2.7.14
+    python-2.7.15
 
 Runtime options include:
 
-- `python-3.6.4`
-- `python-2.7.14`
+- `python-3.7.0`
+- `python-3.6.6`
+- `python-2.7.15`

bin/compile

@@ -15,70 +15,96 @@
 # Fail fast and fail hard.
 set -eo pipefail
 
-# Standard Library.
+# Boostrap the Buildpack Standard Library.
 export BPLOG_PREFIX="buildpack.python"
 export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
 
 [ "$BUILDPACK_XTRACE" ] && set -o xtrace
 
-# Prepend proper path for virtualenv hackery. This will be deprecated soon.
+# Prepend proper path for old-school virtualenv hackery.
+# This may not be neccessary.
 export PATH=:/usr/local/bin:$PATH
 
-# Paths.
+# Setup Path variables, for later use in the Buildpack.
 BIN_DIR=$(cd "$(dirname "$0")"; pwd) # absolute path
 ROOT_DIR=$(dirname "$BIN_DIR")
 BUILD_DIR=$1
 CACHE_DIR=$2
 ENV_DIR=$3
 
+# Export Path variables, for use in sub-scripts.
 export BUILD_DIR CACHE_DIR ENV_DIR
 
+# Set the Buildpack's internet target for downloading Python distributions.
+# The user can provide BUILDPACK_VENDOR_URL to specify a custom target.
+# Note: this is designed for non-Heroku use, as it does not use the user-provided
+# environment variable mechanism (the ENV_DIR).
 VENDOR_URL="https://lang-python.s3.amazonaws.com/$STACK"
 if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then
     VENDOR_URL="$BUILDPACK_VENDOR_URL"
 fi
 export VENDOR_URL
 
-# Python defaults
-DEFAULT_PYTHON_VERSION="python-3.6.4"
-LATEST_3="python-3.6.4"
-LATEST_2="python-2.7.14"
+# Which versions of Python are we using?
+# These variables are used to specify which versions of Python to install by default,
+# as well as prompt the user to upgrade if they are using an un–supported version.
+# Note: When 3.7 lands, I recommend switching to LATEST_36 and LATEST_37.
+DEFAULT_PYTHON_VERSION="python-3.6.6"
+LATEST_3="python-3.6.6"
+LATEST_2="python-2.7.15"
 
+# Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
+# If pip doesn't match this version (the version we install), run the installer.
 PIP_UPDATE="9.0.2"
 
 export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE LATEST_2 LATEST_3
 
-# Common Problem Warnings
+# Common Problem Warnings:
+# This section creates a temporary file in which to stick the output of `pip install`.
+# The `warnings` subscript then greps through this for common problems and guides
+# the user towards resolution of known issues.
 WARNINGS_LOG=$(mktemp)
 export WARNINGS_LOG
 export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
 
-# Setup vendored tools and pip-pop (pip-diff)
+# The buildpack ships with a few executable tools (e.g. pip-grep, etc).
+# This installs them into the path, so we can execute them directly.
 export PATH=$PATH:$ROOT_DIR/vendor/:$ROOT_DIR/vendor/pip-pop
 
-# Support Anvil Build_IDs
+# Set environment variables if they weren't set by the platform.
+# Note: this is legacy, for a deprecated build system known as Anvil.
+# This can likely be removed, with caution.
 [ ! "$SLUG_ID" ] && SLUG_ID="defaultslug"
 [ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID
 [ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK
 
-# Sanitizing environment variables.
+# Sanitize externally-provided environment variables:
+# The following environment variables are either problematic or simply unneccessary
+# for the buildpack to have knowledge of, so we unset them, to keep the environment
+# as clean and pristine as possible.
 unset GIT_DIR PYTHONHOME PYTHONPATH
 unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN
 unset CYTOKINE_LOG_FILE GEM_PATH
 
-# Syntax sugar.
+# Import the utils script, which contains helper functions used throughout the buildpack.
 # shellcheck source=bin/utils
 source "$BIN_DIR/utils"
 
-# Import collection of warnings.
+# Import the warnings script, which contains the `pip install` user warning mechanisms
+# (mentioned and explained above)
 # shellcheck source=bin/warnings
 source "$BIN_DIR/warnings"
 
-# we need to put a bunch of symlinks in there later
+# Make the directory in which we will create symlinks from the temporary build directory
+# to `/app`.
+# Symlinks are required, since Python is not a portable installation.
+# More on this topic later.
 mkdir -p /app/.heroku
 
-# Set up outputs under new context
+# This buildpack programatically generates (or simply copies) a number of files for
+# buildpack machinery: an export script, and a number of `.profile.d` scripts. This
+# section declares the locations of those files and targets.
 PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
 EXPORT_PATH="$BIN_DIR/../export"
 GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh"
@@ -87,43 +113,75 @@ WEB_CONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/WEB_CONCURRENCY.sh"
 # We'll need to send these statics to other scripts we `source`.
 export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH
 
-# Prepend proper environment variables for Python use.
+# Python Environment Variables
+# Set Python-specific environment variables, for running Python within the buildpack.
+# Notes on each variable included.
+
+# PATH is relatively obvious, we need to be able to execute 'python'.
 export PATH=/app/.heroku/python/bin:/app/.heroku/vendor/bin:$PATH
+# Tell Python to not buffer it's stdin/stdout.
 export PYTHONUNBUFFERED=1
+# Set the locale to a well-known and expected standard.
 export LANG=en_US.UTF-8
+# `~/.heroku/vendor` is an place where the buildpack may stick pre-build binaries for known
+# C dependencies (e.g. libmemcached on cedar-14). This section configures Python (GCC, more specifically)
+# and pip to automatically include these paths when building binaries.
 export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
 export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
 export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
 export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
 export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
 
+# The Application Code
+# --------------------
+
 # Switch to the repo's context.
 cd "$BUILD_DIR"
 
-# Prepare the cache.
+# The Cache
+# ---------
+
+# The workflow for the Python Buildpack's cache is as follows:
+#
+# - `~/.heroku/{known-paths}` are copied from the cache into the slug.
+# - The build is executed, modifying `~/.heroku/{known-paths}`.
+# - Once the build is complete, `~/.heroku/{known-paths}` is copied back into the cache.
+
+# Create the cache directory, if it doesn't exist.
 mkdir -p "$CACHE_DIR"
 
 # Restore old artifacts from the cache.
 mkdir -p .heroku
 
+# The Python installation.
 cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true
+# A plain text file which contains the current stack being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
+# A plain text file which contains the current python version being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
+# Any pre-compiled binaries, provided by the buildpack.
 cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true
+# "editable" installations of code repositories, via pip or pipenv.
 if [[ -d "$CACHE_DIR/.heroku/src" ]]; then
   cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true
 fi
 
-# Experimental pre_compile hook.
+# The pre_compile hook. Customers rely on this. Don't remove it.
+# This part of the code is used to allow users to customize their build experience
+# without forking the buildpack by providing a `bin/pre_compile` script, which gets
+# run inline with the buildpack automatically.
+
 # shellcheck source=bin/steps/hooks/pre_compile
 source "$BIN_DIR/steps/hooks/pre_compile"
 
-# Sticky runtimes.
+# Sticky runtimes. If there was a previous build, and it used a given version of Python,
+# continue to use that version of Python in perpituity (warnings will be raised if
+# they are out–of–date).
 if [ -f "$CACHE_DIR/.heroku/python-version" ]; then
   DEFAULT_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
 fi
 
-# Stack fallback for non-declared caches.
+# We didn't always record the stack version. This code is in place because of that.
 if [ -f "$CACHE_DIR/.heroku/python-stack" ]; then
   CACHED_PYTHON_STACK=$(cat "$CACHE_DIR/.heroku/python-stack")
 else
@@ -133,17 +191,26 @@ fi
 export CACHED_PYTHON_STACK
 
 # Pipenv Python version support.
+# Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version).
+# Convert it to a runtime.txt file.
+
 # shellcheck source=bin/steps/pipenv-python-version
 source "$BIN_DIR/steps/pipenv-python-version"
 
-# If no runtime given, assume default version.
+# If no runtime was provided by the user, assume the default Python runtime version.
 if [ ! -f runtime.txt ]; then
   echo "$DEFAULT_PYTHON_VERSION" > runtime.txt
 fi
 
+# Create the directory for .profile.d, if it doesn't exist.
 mkdir -p "$(dirname "$PROFILE_PATH")"
+# Create the directory for editable source code installation, if it doesn't exist.
 mkdir -p /app/.heroku/src
 
+# On Heroku CI, builds happen in `/app`. Otherwise, on the Heroku platform,
+# they occur in a temp directory. Beacuse Python is not portable, we must create
+# symlinks to emulate that we are operating in `/app` during the build process.
+# This is (hopefully obviously) because apps end up running from `/app` in production.
 if [[ $BUILD_DIR != '/app' ]]; then
     # python expects to reside in /app, so set up symlinks
     # we will not remove these later so subsequent buildpacks can still invoke it
@@ -152,60 +219,79 @@ if [[ $BUILD_DIR != '/app' ]]; then
     # Note: .heroku/src is copied in later.
 fi
 
-# Install Python.
-let start=$(nowms)
+# Download / Install Python, from pre-build binaries available on Amazon S3.
+# This step also bootstraps pip / setuptools.
+(( start=$(nowms) ))
 # shellcheck source=bin/steps/python
 source "$BIN_DIR/steps/python"
 mtime "python.install.time" "${start}"
 
-# Pipenv support.
+# Install Pipenv dependencies, if a Pipfile was provided.
 # shellcheck source=bin/steps/pipenv
 source "$BIN_DIR/steps/pipenv"
 
 # Uninstall removed dependencies with Pip.
-let start=$(nowms)
+# The buildpack will automatically remove any declared dependencies (in requirements.txt)
+# that were explicitly removed. This machinery is a bit complex, but it is not complicated.
+(( start=$(nowms) ))
 # shellcheck source=bin/steps/pip-uninstall
 source "$BIN_DIR/steps/pip-uninstall"
 mtime "pip.uninstall.time" "${start}"
 
-
 # If no requirements.txt file given, assume `setup.py develop` is intended.
+# This allows for people to ship a setup.py application to Heroku
+# (which is rare, but I vouch that it should work!)
+
 if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
   echo "-e ." > requirements.txt
 fi
 
 # Fix egg-links.
+# Because we're installing things into a different path than we're running them (temp dir vs app dir),
+# We must re-write all of Python's eggpath links to target the proper directory.
 # shellcheck source=bin/steps/eggpath-fix
 source "$BIN_DIR/steps/eggpath-fix"
 
 # Mercurial support.
+# If a customer appears to be using mercurial for dependency resolution, we install it first.
+# Note: this only applies to pip, not pipenv. This can likely be removed, over time. Measure it first.
 # shellcheck source=bin/steps/mercurial
 source "$BIN_DIR/steps/mercurial"
 
 # Pylibmc support.
+# On cedar-14, libmemcached was not available. The buildpack provides its own version, instead.
 # shellcheck source=bin/steps/pylibmc
 source "$BIN_DIR/steps/pylibmc"
 
-# Support for Geo libraries.
+# Support for Geo libraries. This is deprecated functionality, only functional on cedar-14.
+# It is undocumented.
 # shellcheck source=bin/steps/geo-libs
 sub_env "$BIN_DIR/steps/geo-libs"
 
 # GDAL support.
+# This is part of the Geo support.
 # shellcheck source=bin/steps/gdal
 source "$BIN_DIR/steps/gdal"
 
-# Install dependencies with Pip (where the magic happens).
-let start=$(nowms)
+# pip install
+# -----------
+
+# Install dependencies with pip (where the magic happens).
+(( start=$(nowms) ))
 # shellcheck source=bin/steps/pip-install
 source "$BIN_DIR/steps/pip-install"
 mtime "pip.install.time" "${start}"
 
 # Support for NLTK corpora.
-let start=$(nowms)
+# Note: this may only work on Python 2.7. I don't think many customers use this functionality,
+# and it should probably be undocumented.
+# (there's an import error on 3.6 that should hopefully be fixed upstream at some point)
+(( start=$(nowms) ))
 sub_env "$BIN_DIR/steps/nltk"
 mtime "nltk.download.time" "${start}"
 
-# Support for pip install -e.
+# Support for editable installations. Here, we are copying pip–created src directory,
+# and copying it into the proper place (the logical place to do this was early, but it must be done here).
 # In CI, $BUILD_DIR is /app.
 if [[ ! "$BUILD_DIR" == "/app" ]]; then
   rm -fr "$BUILD_DIR/.heroku/src"
@@ -214,24 +300,35 @@ fi
 
 
 # Django collectstatic support.
-let start=$(nowms)
+# The buildpack automatically runs collectstatic for Django applications.
+# This is the cause for the majority of build failures on the Python platform.
+# These failures are intentional — if collectstatic (which can be tricky, at times) fails,
+# your build fails.
+(( start=$(nowms) ))
 sub_env "$BIN_DIR/steps/collectstatic"
 mtime "collectstatic.time" "${start}"
 
-# Create .profile script for application runtime environment variables.
-set_env PATH "\$HOME/.heroku/python/bin:\$PATH"
-set_env PYTHONUNBUFFERED true
-set_env PYTHONHOME "\$HOME/.heroku/python"
 
+# Progamatically create .profile.d script for application runtime environment variables.
+
+# Set the PATH to include Python / pip / pipenv / etc.
+set_env PATH "\$HOME/.heroku/python/bin:\$PATH"
+# Tell Python to run in unbuffered mode.
+set_env PYTHONUNBUFFERED true
+# Tell Python where it lives.
+set_env PYTHONHOME "\$HOME/.heroku/python"
+# Set variables for C libraries.
 set_env LIBRARY_PATH "\$HOME/.heroku/vendor/lib:\$HOME/.heroku/python/lib:\$LIBRARY_PATH"
 set_env LD_LIBRARY_PATH "\$HOME/.heroku/vendor/lib:\$HOME/.heroku/python/lib:\$LD_LIBRARY_PATH"
-
+# Locale.
 set_default_env LANG en_US.UTF-8
+# The Python hash seed is set to random.
 set_default_env PYTHONHASHSEED random
+# Tell Python to look for Python modules in the /app dir. Don't change this.
 set_default_env PYTHONPATH "\$HOME"
 
-# python expects to be in /app, if at runtime, it is not, set
-# up symlinks... this can occur when the subdir buildpack is used
+# Python expects to be in /app, if at runtime, it is not, set
+# up symlinks… this can occur when the subdir buildpack is used.
 cat <<EOT >> "$PROFILE_PATH"
 if [[ \$HOME != "/app" ]]; then
     mkdir -p /app/.heroku
@@ -244,8 +341,7 @@ EOT
 cp "$ROOT_DIR/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH"
 cp "$ROOT_DIR/vendor/python.gunicorn.sh" "$GUNICORN_PROFILE_PATH"
 
-
-# Experimental post_compile hook.
+# Experimental post_compile hook. Don't remove this.
 # shellcheck source=bin/steps/hooks/post_compile
 source "$BIN_DIR/steps/hooks/post_compile"
 
@@ -253,8 +349,7 @@ source "$BIN_DIR/steps/hooks/post_compile"
 # shellcheck source=bin/steps/eggpath-fix2
 source "$BIN_DIR/steps/eggpath-fix2"
 
-# Store new artifacts in cache.
-
+# Store new artifacts in the cache.
 rm -rf "$CACHE_DIR/.heroku/python"
 rm -rf "$CACHE_DIR/.heroku/python-version"
 rm -rf "$CACHE_DIR/.heroku/python-stack"

bin/steps/collectstatic

@@ -11,7 +11,7 @@
 #   - $DEBUG_COLLECTSTATIC: upon failure, print out environment variables.
 
 # shellcheck source=bin/utils
-source $BIN_DIR/utils
+source "$BIN_DIR/utils"
 
 # Location of 'manage.py', if it exists.
 MANAGE_FILE=$(find . -maxdepth 3 -type f -name 'manage.py' -printf '%d\t%P\n' | sort -nk1 | cut -f2 | head -1)
@@ -44,22 +44,20 @@ if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALL
     # Display a warning if collectstatic failed.
     [ "$COLLECTSTATIC_STATUS" -ne 0 ] && {
 
-        mcount "failure.collectstatic"
-
         if grep -q 'SyntaxError' "$COLLECTSTATIC_LOG"; then
             mcount "failure.collectstatic.syntax-error"
-        fi
 
-        if grep -q 'ImproperlyConfigured' "$COLLECTSTATIC_LOG"; then
+        elif grep -q 'ImproperlyConfigured' "$COLLECTSTATIC_LOG"; then
             mcount "failure.collectstatic.improper-configuration"
-        fi
 
-        if grep -q 'The CSS file' "$COLLECTSTATIC_LOG"; then
+        elif grep -q 'The CSS file' "$COLLECTSTATIC_LOG"; then
             mcount "failure.collectstatic.fancy-references"
-        fi
 
-        if grep -q 'OSError' "$COLLECTSTATIC_LOG"; then
+        elif grep -q 'OSError' "$COLLECTSTATIC_LOG"; then
             mcount "failure.collectstatic.missing-file"
+
+        else
+            mcount "failure.collectstatic.other"
         fi
 
         echo

bin/steps/pip-install

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # shellcheck source=bin/utils
-source $BIN_DIR/utils
+source "$BIN_DIR/utils"
 
 if [ ! "$SKIP_PIP_INSTALL" ]; then
 
@@ -20,9 +20,6 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     mcount "tool.pip"
 
     # Count expected build failures.
-    if grep -q 'wsgiref' requirements.txt; then
-        mcount "failure.wsgiref"
-    fi
     if grep -q '==0.0.0' requirements.txt; then
         mcount "failure.none-version"
     fi
@@ -35,10 +32,10 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     show-warnings
 
     if [[ ! $PIP_STATUS -eq 0 ]]; then
+        mcount "failure.pip-install"
         exit 1
     fi
 
-
     # Smart Requirements handling
     cp requirements.txt .heroku/python/requirements-declared.txt
     /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt

bin/steps/pip-uninstall

@@ -3,7 +3,7 @@
 set +e
 # Install dependencies with Pip.
 # shellcheck source=bin/utils
-source $BIN_DIR/utils
+source "$BIN_DIR/utils"
 
 if [ ! "$SKIP_PIP_INSTALL" ]; then
 
@@ -20,7 +20,7 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
 
     if [[ -s .heroku/python/requirements-stale.txt ]]; then
       puts-step "Uninstalling stale dependencies"
-      /app/.heroku/python/bin/pip uninstall -r .heroku/python/requirements-stale.txt -y --exists-action=w | cleanup | indent
+      /app/.heroku/python/bin/pip uninstall -r .heroku/python/requirements-stale.txt -y --exists-action=w --disable-pip-version-check | cleanup | indent
     fi
   fi
 

bin/steps/pipenv

@@ -3,7 +3,7 @@
 # export CLINT_FORCE_COLOR=1
 # export PIPENV_FORCE_COLOR=1
 # shellcheck source=bin/utils
-source $BIN_DIR/utils
+source "$BIN_DIR/utils"
 set -e
 
 if [[ -f Pipfile.lock ]]; then
@@ -41,7 +41,7 @@ if [ ! "$SKIP_PIPENV_INSTALL" ]; then
             export PIP_EXTRA_INDEX_URL
         fi
 
-        export PIPENV_VERSION="11.8.2"
+        export PIPENV_VERSION="2018.5.18"
 
         # Install pipenv.
         /app/.heroku/python/bin/pip install pipenv==$PIPENV_VERSION --upgrade &> /dev/null

bin/steps/pylibmc

@@ -9,7 +9,7 @@
 #
 # This script is invoked by [`bin/compile`](/).
 
-if [[ "$STACK" == "heroku-16" ]]; then
+if [[ "$STACK" != "cedar-14" ]]; then
     # libmemcached is pre-installed in the stack image so there is no need to vendor it.
     return 0
 fi

bin/steps/python

@@ -67,7 +67,8 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
   rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
   rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
 
-  /app/.heroku/python/bin/python "$ROOT_DIR/vendor/get-pip.py" &> /dev/null
+  /app/.heroku/python/bin/python "$ROOT_DIR/vendor/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null
+  /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
 
 fi
 

buildpack.toml

@@ -0,0 +1,13 @@
+[buildpack]
+name = "Python"
+
+  [publish.Ignore]
+  files = [
+    "test/",
+    ".gitignore",
+    ".dockerignore",
+    ".github/",
+    "Dockerfile",
+    "Pipfile",
+    "Pipfile.lock"
+  ]

builds/runtimes/python-2.7.15

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/2.7.15/Python-2.7.15.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-2.7.15 src
+cd src
+
+./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +

builds/runtimes/python-3.6.5

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.6.5/Python-3.6.5.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.6.5 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python

builds/runtimes/python-3.6.6

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.6.6/Python-3.6.6.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.6.6 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python

builds/runtimes/python-3.7.0

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.7.0/Python-3.7.0.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.7.0 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python

test/fixtures/git-egg/requirements.txt

@@ -0,0 +1 @@
+-e git+https://github.com/requests/requests.git#egg=requests

test/fixtures/pipenv-lock/Pipfile.lock

@@ -1,27 +1,14 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "397f2c55e3558ea57d292e3fc19b34e483770e5ec02cdedfb1f330680cd26635"
-        },
-        "host-environment-markers": {
-            "implementation_name": "cpython",
-            "implementation_version": "3.6.3",
-            "os_name": "posix",
-            "platform_machine": "x86_64",
-            "platform_python_implementation": "CPython",
-            "platform_release": "16.7.0",
-            "platform_system": "Darwin",
-            "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64",
-            "python_full_version": "3.6.3",
-            "python_version": "3.6",
-            "sys_platform": "darwin"
+            "sha256": "09ad9dcae1870ba083f43c5a05ed8943b23bd4c27e61a13ecf4e16d18500ad98"
         },
         "pipfile-spec": 6,
         "requires": {},
         "sources": [
             {
                 "name": "pypi",
-                "url": "https://pypi.python.org/simple",
+                "url": "https://pypi.org/simple",
                 "verify_ssl": true
             }
         ]
@@ -29,22 +16,23 @@
     "default": {
         "delegator.py": {
             "hashes": [
-                "sha256:2575c4adc923ad0b8fdaa433f862b2b7cf21982717fb23cc895fd8f249ea820c",
-                "sha256:495e11ada66648650171a6c9a188df4eb050b235abff8771f41ee8a064eb9ded"
+                "sha256:2d46966a7f484d271b09e2646eae1e9acadc4fdf2cb760c142f073e81c927d8d",
+                "sha256:58f3ea6fe36680e1d828e2e66e52844b826f186409dfee4436e42351b0e699fe"
             ],
-            "version": "==0.0.13"
+            "index": "pypi",
+            "version": "==0.1.0"
         },
         "pexpect": {
             "hashes": [
-                "sha256:f853b52afaf3b064d29854771e2db509ef80392509bde2dd7a6ecf2dfc3f0018",
-                "sha256:3d132465a75b57aa818341c6521392a06cc660feb3988d7f1074f39bd23c9a92"
+                "sha256:9783f4644a3ef8528a6f20374eeb434431a650c797ca6d8df0d81e30fffdfa24",
+                "sha256:9f8eb3277716a01faafaba553d629d3d60a1a624c7cf45daa600d2148c30020c"
             ],
-            "version": "==4.2.1"
+            "version": "==4.5.0"
         },
         "ptyprocess": {
             "hashes": [
-                "sha256:e8c43b5eee76b2083a9badde89fd1bbce6c8942d1045146e100b7b5e014f4f1a",
-                "sha256:e64193f0047ad603b71f202332ab5527c5e52aa7c8b609704fc28c0dc20c4365"
+                "sha256:e64193f0047ad603b71f202332ab5527c5e52aa7c8b609704fc28c0dc20c4365",
+                "sha256:e8c43b5eee76b2083a9badde89fd1bbce6c8942d1045146e100b7b5e014f4f1a"
             ],
             "version": "==0.5.2"
         }

test/fixtures/python2/runtime.txt

@@ -1 +1 @@
-python-2.7.14
+python-2.7.15

test/run

@@ -12,7 +12,7 @@ testPipenvLock() {
 
 testPipenvVersion() {
   compile "pipenv-version"
-  assertCaptured "3.6.4"
+  assertCaptured "3.6.6"
   assertCapturedSuccess
 }
 
@@ -77,13 +77,19 @@ testPylibmc() {
 
 testPython2() {
   compile "python2"
-  assertCaptured "python-2.7.14"
+  assertCaptured "python-2.7.15"
   assertCapturedSuccess
 }
 
 testPython3() {
   compile "python3"
-  assertCaptured "python-3.6.4"
+  assertCaptured "python-3.6.6"
+  assertCapturedSuccess
+}
+
+testGitEgg() {
+  compile "git-egg"
+  assertCaptured "requests"
   assertCapturedSuccess
 }
 

vendor/pipenv-to-pip

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import io
 import json
 import sys
 
@@ -7,7 +8,7 @@ import sys
 def main():
     INFILE = sys.argv[1]
 
-    with open(INFILE, 'rb') as f:
+    with io.open(INFILE, 'r', encoding='utf-8') as f:
         lockfile = json.load(f)
 
     packages = []