Merge pull request #759 from heroku/update-travis

Update travis setup for Hatchet

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @heroku/languages

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,8 @@
+<!-- Hi and welcome to the Heroku Python buildpack repository!
+
+If you meant to open a PR against a fork instead of upstream, please adjust the base branch:
+https://help.github.com/articles/changing-the-base-branch-of-a-pull-request/
+
+Otherwise thank you in advance for your Pull Request - just remember to
+include as much information as possible to help the reviewers :-)
+-->

.gitignore

@@ -3,3 +3,7 @@ site
 .DS_Store
 
 /.envrc
+repos/*
+
+#Venv
+buildpack/*

.travis.yml

@@ -1,23 +1,39 @@
-language: bash
+language: ruby
 dist: trusty
+sudo: required
+rvm:
+- 2.4.4
+before_script:
+  - gem install bundler -v 1.16.2
+  - bundle exec rake hatchet:setup_travis
+before_install:
+  - sudo bash etc/ci-setup.sh
 jobs:
   include:
-    - stage: "Bash linting (shellcheck)"
+  - stage: Bash linting (shellcheck)
-      sudo: false
+    sudo: false
-      addons:
+    before_install:
-        apt:
+    - wget -c https://goo.gl/ZzKHFv -O - | tar -xvJ -C /tmp/
-          sources:
+    - PATH="/tmp/shellcheck-latest:$PATH"
-          - debian-sid    # Grab shellcheck from the Debian repo (o_O)
+    script: make check
-          packages:
+  - stage: Stack Unit Tests
-          - shellcheck
+    services: docker
-      script: make check
+    env: STACK=heroku-18
-
+    script: "./tests.sh"
-    - stage: "Stack Tests"
+  - stage: Stack Unit Tests
-      services: docker
+    services: docker
-      env: STACK=heroku-16
+    env: STACK=heroku-16
-      script: ./tests.sh
+    script: "./tests.sh"
-
+  - stage: Stack Unit Tests
-    - stage: "Stack Tests"
+    services: docker
-      services: docker
+    env: STACK=cedar-14
-      env: STACK=cedar-14
+    script: "./tests.sh"
-      script: ./tests.sh
+  - stage: Hatchet Integration
+    script: "bundle exec rspec"
+env:
+  global:
+  - HATCHET_RETRIES=3
+  - IS_RUNNING_ON_CI=true
+  - HATCHET_APP_LIMIT=5
+  - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
+  - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=

CHANGELOG.md

@@ -1,5 +1,41 @@
 # Python Buildpack Changelog
 
+# 139
+
+Improvements to Python install messaging
+
+# 138 (2018-08-01)
+
+Use stack image SQLite3 instead of vendoring
+
+# 137 (2018-07-17)
+
+Prevent 3.7.0 from appearing as unsupported in buildpack messaging.
+
+# 136 (2018-06-28)
+
+Upgrade to 3.6.6 and support 3.7.0 on all runtimes.
+
+# 135 (2018-05-29)
+
+Upgrade Pipenv to v2018.5.18.
+
+# 134 (2018-05-02)
+
+Default to 3.6.5, bugfixes.
+
+# 133
+
+Fixes for Pip 10 release.
+
+# 132
+
+Improve pip installation, with the release of v9.0.2.
+
+# 131
+
+Fix bug with pip.
+
 # 130
 
 Better upgrade strategy for pip.

Dockerfile.heroku-18

@@ -0,0 +1,13 @@
+FROM heroku/heroku:18-build
+
+WORKDIR /app
+ENV WORKSPACE_DIR="/app/builds" \
+    S3_BUCKET="lang-python" \
+    S3_PREFIX="heroku-18/"
+
+RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/
+RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+
+COPY . /app

Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+gem "rspec"
+gem "heroku_hatchet"
+gem "rspec-retry"
+gem "rake"

Gemfile.lock

@@ -0,0 +1,71 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    concurrent-ruby (1.0.5)
+    diff-lcs (1.3)
+    erubis (2.7.0)
+    excon (0.62.0)
+    heroics (0.0.24)
+      erubis (~> 2.0)
+      excon
+      moneta
+      multi_json (>= 1.9.2)
+    heroku_hatchet (4.0.2)
+      excon (~> 0)
+      minitest-retry (~> 0.1.9)
+      platform-api (~> 2)
+      repl_runner (~> 0.0.3)
+      rrrretry (~> 1)
+      thor (~> 0)
+      threaded (~> 0)
+    i18n (1.1.0)
+      concurrent-ruby (~> 1.0)
+    minitest (5.11.3)
+    minitest-retry (0.1.9)
+      minitest (>= 5.0)
+    moneta (0.8.1)
+    multi_json (1.13.1)
+    platform-api (2.1.0)
+      heroics (~> 0.0.23)
+      moneta (~> 0.8.1)
+    rake (12.3.1)
+    repl_runner (0.0.3)
+      activesupport
+    rrrretry (1.0.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-retry (0.6.1)
+      rspec-core (> 3.3)
+    rspec-support (3.8.0)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    threaded (0.0.4)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  heroku_hatchet
+  rake
+  rspec
+  rspec-retry
+
+BUNDLED WITH
+   1.16.3

Makefile

@@ -18,6 +18,11 @@ test-heroku-16:
 	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
 	@echo ""
 
+test-heroku-18:
+	@echo "Running tests in docker (heroku-18)..."
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@echo ""
+
 buildenv-heroku-16:
 	@echo "Creating build environment (heroku-16)..."
 	@echo

README.md

@@ -23,9 +23,9 @@ Deploying a Python application couldn't be easier:
     $ git push heroku master
     …
     -----> Python app detected
-    -----> Installing python-3.6.4
+    -----> Installing python-3.6.6
     -----> Installing pip
-    -----> Installing requirements with Pipenv 11.7.1…
+    -----> Installing requirements with Pipenv 2018.5.18…
            ...
            Installing dependencies from Pipfile…
     -----> Discovering process types
@@ -49,14 +49,15 @@ Specific versions of the Python runtime can be specified in your `Pipfile`:
 Or, more specifically:
 
     [requires]
-    python_full_version = "2.7.14"
+    python_full_version = "2.7.15"
 
 Or, with a `runtime.txt` file:
 
     $ cat runtime.txt
-    python-2.7.14
+    python-2.7.15
 
 Runtime options include:
 
-- `python-3.6.4`
+- `python-3.7.0`
-- `python-2.7.14`
+- `python-3.6.6`
+- `python-2.7.15`

Rakefile

@@ -0,0 +1 @@
+require 'hatchet/tasks'

bin/compile

@@ -15,70 +15,97 @@
 # Fail fast and fail hard.
 set -eo pipefail
 
-# Standard Library.
+# Boostrap the Buildpack Standard Library.
 export BPLOG_PREFIX="buildpack.python"
 export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
 
 [ "$BUILDPACK_XTRACE" ] && set -o xtrace
 
-# Prepend proper path for virtualenv hackery. This will be deprecated soon.
+# Prepend proper path for old-school virtualenv hackery.
+# This may not be neccessary.
 export PATH=:/usr/local/bin:$PATH
 
-# Paths.
+# Setup Path variables, for later use in the Buildpack.
 BIN_DIR=$(cd "$(dirname "$0")"; pwd) # absolute path
 ROOT_DIR=$(dirname "$BIN_DIR")
 BUILD_DIR=$1
 CACHE_DIR=$2
 ENV_DIR=$3
 
+# Export Path variables, for use in sub-scripts.
 export BUILD_DIR CACHE_DIR ENV_DIR
 
+# Set the Buildpack's internet target for downloading Python distributions.
+# The user can provide BUILDPACK_VENDOR_URL to specify a custom target.
+# Note: this is designed for non-Heroku use, as it does not use the user-provided
+# environment variable mechanism (the ENV_DIR).
 VENDOR_URL="https://lang-python.s3.amazonaws.com/$STACK"
 if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then
     VENDOR_URL="$BUILDPACK_VENDOR_URL"
 fi
 export VENDOR_URL
 
-# Python defaults
+# Which versions of Python are we using?
-DEFAULT_PYTHON_VERSION="python-3.6.4"
+# These variables are used to specify which versions of Python to install by default,
-LATEST_3="python-3.6.4"
+# as well as prompt the user to upgrade if they are using an un–supported version.
-LATEST_2="python-2.7.14"
+# Note: When 3.7 lands, I recommend switching to LATEST_36 and LATEST_37.
+DEFAULT_PYTHON_VERSION="python-3.6.6"
+LATEST_36="python-3.6.6"
+LATEST_37="python-3.7.0"
+LATEST_2="python-2.7.15"
 
+# Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
-PIP_UPDATE="9.0.1"
+# If pip doesn't match this version (the version we install), run the installer.
+PIP_UPDATE="9.0.2"
 
-export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE LATEST_2 LATEST_3
+export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE LATEST_2 LATEST_36 LATEST_37
 
-# Common Problem Warnings
+# Common Problem Warnings:
+# This section creates a temporary file in which to stick the output of `pip install`.
+# The `warnings` subscript then greps through this for common problems and guides
+# the user towards resolution of known issues.
 WARNINGS_LOG=$(mktemp)
 export WARNINGS_LOG
 export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
 
-# Setup vendored tools and pip-pop (pip-diff)
+# The buildpack ships with a few executable tools (e.g. pip-grep, etc).
+# This installs them into the path, so we can execute them directly.
 export PATH=$PATH:$ROOT_DIR/vendor/:$ROOT_DIR/vendor/pip-pop
 
-# Support Anvil Build_IDs
+# Set environment variables if they weren't set by the platform.
+# Note: this is legacy, for a deprecated build system known as Anvil.
+# This can likely be removed, with caution.
 [ ! "$SLUG_ID" ] && SLUG_ID="defaultslug"
 [ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID
 [ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK
 
-# Sanitizing environment variables.
+# Sanitize externally-provided environment variables:
+# The following environment variables are either problematic or simply unneccessary
+# for the buildpack to have knowledge of, so we unset them, to keep the environment
+# as clean and pristine as possible.
 unset GIT_DIR PYTHONHOME PYTHONPATH
 unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN
 unset CYTOKINE_LOG_FILE GEM_PATH
 
-# Syntax sugar.
+# Import the utils script, which contains helper functions used throughout the buildpack.
 # shellcheck source=bin/utils
 source "$BIN_DIR/utils"
 
-# Import collection of warnings.
+# Import the warnings script, which contains the `pip install` user warning mechanisms
+# (mentioned and explained above)
 # shellcheck source=bin/warnings
 source "$BIN_DIR/warnings"
 
-# we need to put a bunch of symlinks in there later
+# Make the directory in which we will create symlinks from the temporary build directory
+# to `/app`.
+# Symlinks are required, since Python is not a portable installation.
+# More on this topic later.
 mkdir -p /app/.heroku
 
-# Set up outputs under new context
+# This buildpack programatically generates (or simply copies) a number of files for
+# buildpack machinery: an export script, and a number of `.profile.d` scripts. This
+# section declares the locations of those files and targets.
 PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
 EXPORT_PATH="$BIN_DIR/../export"
 GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh"
@@ -87,43 +114,77 @@ WEB_CONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/WEB_CONCURRENCY.sh"
 # We'll need to send these statics to other scripts we `source`.
 export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH
 
-# Prepend proper environment variables for Python use.
+# Python Environment Variables
+# Set Python-specific environment variables, for running Python within the buildpack.
+# Notes on each variable included.
+
+# PATH is relatively obvious, we need to be able to execute 'python'.
 export PATH=/app/.heroku/python/bin:/app/.heroku/vendor/bin:$PATH
+# Tell Python to not buffer it's stdin/stdout.
 export PYTHONUNBUFFERED=1
+# Set the locale to a well-known and expected standard.
 export LANG=en_US.UTF-8
+# `~/.heroku/vendor` is an place where the buildpack may stick pre-build binaries for known
+# C dependencies (e.g. libmemcached on cedar-14). This section configures Python (GCC, more specifically)
+# and pip to automatically include these paths when building binaries.
 export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
 export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
 export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
 export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
 export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
 
+# The Application Code
+# --------------------
+
 # Switch to the repo's context.
 cd "$BUILD_DIR"
 
-# Prepare the cache.
+# The Cache
+# ---------
+
+# The workflow for the Python Buildpack's cache is as follows:
+#
+# - `~/.heroku/{known-paths}` are copied from the cache into the slug.
+# - The build is executed, modifying `~/.heroku/{known-paths}`.
+# - Once the build is complete, `~/.heroku/{known-paths}` is copied back into the cache.
+
+# Create the cache directory, if it doesn't exist.
 mkdir -p "$CACHE_DIR"
 
 # Restore old artifacts from the cache.
 mkdir -p .heroku
 
+# The Python installation.
 cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true
+# A plain text file which contains the current stack being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
+# A plain text file which contains the current python version being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
+# A plain text file which contains the current sqlite3 version being used (used for cache busting).
+cp -R "$CACHE_DIR/.heroku/python-sqlite3-version" .heroku/ &> /dev/null || true
+# Any pre-compiled binaries, provided by the buildpack.
 cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true
+# "editable" installations of code repositories, via pip or pipenv.
 if [[ -d "$CACHE_DIR/.heroku/src" ]]; then
   cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true
 fi
 
-# Experimental pre_compile hook.
+# The pre_compile hook. Customers rely on this. Don't remove it.
+# This part of the code is used to allow users to customize their build experience
+# without forking the buildpack by providing a `bin/pre_compile` script, which gets
+# run inline with the buildpack automatically.
+
 # shellcheck source=bin/steps/hooks/pre_compile
 source "$BIN_DIR/steps/hooks/pre_compile"
 
-# Sticky runtimes.
+# Sticky runtimes. If there was a previous build, and it used a given version of Python,
+# continue to use that version of Python in perpituity (warnings will be raised if
+# they are out–of–date).
 if [ -f "$CACHE_DIR/.heroku/python-version" ]; then
   DEFAULT_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
 fi
 
-# Stack fallback for non-declared caches.
+# We didn't always record the stack version. This code is in place because of that.
 if [ -f "$CACHE_DIR/.heroku/python-stack" ]; then
   CACHED_PYTHON_STACK=$(cat "$CACHE_DIR/.heroku/python-stack")
 else
@@ -133,17 +194,26 @@ fi
 export CACHED_PYTHON_STACK
 
 # Pipenv Python version support.
+# Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version).
+# Convert it to a runtime.txt file.
+
 # shellcheck source=bin/steps/pipenv-python-version
 source "$BIN_DIR/steps/pipenv-python-version"
 
-# If no runtime given, assume default version.
+# If no runtime was provided by the user, assume the default Python runtime version.
 if [ ! -f runtime.txt ]; then
   echo "$DEFAULT_PYTHON_VERSION" > runtime.txt
 fi
 
+# Create the directory for .profile.d, if it doesn't exist.
 mkdir -p "$(dirname "$PROFILE_PATH")"
+# Create the directory for editable source code installation, if it doesn't exist.
 mkdir -p /app/.heroku/src
 
+# On Heroku CI, builds happen in `/app`. Otherwise, on the Heroku platform,
+# they occur in a temp directory. Beacuse Python is not portable, we must create
+# symlinks to emulate that we are operating in `/app` during the build process.
+# This is (hopefully obviously) because apps end up running from `/app` in production.
 if [[ $BUILD_DIR != '/app' ]]; then
     # python expects to reside in /app, so set up symlinks
     # we will not remove these later so subsequent buildpacks can still invoke it
@@ -152,60 +222,89 @@ if [[ $BUILD_DIR != '/app' ]]; then
     # Note: .heroku/src is copied in later.
 fi
 
-# Install Python.
+# Download / Install Python, from pre-build binaries available on Amazon S3.
-let start=$(nowms)
+# This step also bootstraps pip / setuptools.
+(( start=$(nowms) ))
 # shellcheck source=bin/steps/python
 source "$BIN_DIR/steps/python"
 mtime "python.install.time" "${start}"
 
-# Pipenv support.
+# Install Pipenv dependencies, if a Pipfile was provided.
 # shellcheck source=bin/steps/pipenv
 source "$BIN_DIR/steps/pipenv"
 
 # Uninstall removed dependencies with Pip.
-let start=$(nowms)
+# The buildpack will automatically remove any declared dependencies (in requirements.txt)
+# that were explicitly removed. This machinery is a bit complex, but it is not complicated.
+(( start=$(nowms) ))
 # shellcheck source=bin/steps/pip-uninstall
 source "$BIN_DIR/steps/pip-uninstall"
 mtime "pip.uninstall.time" "${start}"
 
-
 # If no requirements.txt file given, assume `setup.py develop` is intended.
+# This allows for people to ship a setup.py application to Heroku
+# (which is rare, but I vouch that it should work!)
+
 if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
   echo "-e ." > requirements.txt
 fi
 
 # Fix egg-links.
+# Because we're installing things into a different path than we're running them (temp dir vs app dir),
+# We must re-write all of Python's eggpath links to target the proper directory.
 # shellcheck source=bin/steps/eggpath-fix
 source "$BIN_DIR/steps/eggpath-fix"
 
 # Mercurial support.
+# If a customer appears to be using mercurial for dependency resolution, we install it first.
+# Note: this only applies to pip, not pipenv. This can likely be removed, over time. Measure it first.
 # shellcheck source=bin/steps/mercurial
 source "$BIN_DIR/steps/mercurial"
 
 # Pylibmc support.
+# On cedar-14, libmemcached was not available. The buildpack provides its own version, instead.
 # shellcheck source=bin/steps/pylibmc
 source "$BIN_DIR/steps/pylibmc"
 
-# Support for Geo libraries.
+# Support for Geo libraries. This is deprecated functionality, only functional on cedar-14.
+# It is undocumented.
 # shellcheck source=bin/steps/geo-libs
 sub_env "$BIN_DIR/steps/geo-libs"
 
 # GDAL support.
+# This is part of the Geo support.
 # shellcheck source=bin/steps/gdal
 source "$BIN_DIR/steps/gdal"
 
-# Install dependencies with Pip (where the magic happens).
+# SQLite3 support.
-let start=$(nowms)
+# This sets up and installs sqlite3 dev headers and the sqlite3 binary but not the
+# libsqlite3-0 library since that exists on the stack image.
+# Note: This only applies to Python 2.7.15+ and Python 3.6.6+
+(( start=$(nowms) ))
+# shellcheck source=bin/steps/sqlite3
+source "$BIN_DIR/steps/sqlite3"
+buildpack_sqlite3_install
+mtime "sqlite3.install.time" "${start}"
+
+# pip install
+# -----------
+
+# Install dependencies with pip (where the magic happens).
+(( start=$(nowms) ))
 # shellcheck source=bin/steps/pip-install
 source "$BIN_DIR/steps/pip-install"
 mtime "pip.install.time" "${start}"
 
 # Support for NLTK corpora.
-let start=$(nowms)
+# Note: this may only work on Python 2.7. I don't think many customers use this functionality,
+# and it should probably be undocumented.
+# (there's an import error on 3.6 that should hopefully be fixed upstream at some point)
+(( start=$(nowms) ))
 sub_env "$BIN_DIR/steps/nltk"
 mtime "nltk.download.time" "${start}"
 
-# Support for pip install -e.
+# Support for editable installations. Here, we are copying pip–created src directory,
+# and copying it into the proper place (the logical place to do this was early, but it must be done here).
 # In CI, $BUILD_DIR is /app.
 if [[ ! "$BUILD_DIR" == "/app" ]]; then
   rm -fr "$BUILD_DIR/.heroku/src"
@@ -214,24 +313,35 @@ fi
 
 
 # Django collectstatic support.
-let start=$(nowms)
+# The buildpack automatically runs collectstatic for Django applications.
+# This is the cause for the majority of build failures on the Python platform.
+# These failures are intentional — if collectstatic (which can be tricky, at times) fails,
+# your build fails.
+(( start=$(nowms) ))
 sub_env "$BIN_DIR/steps/collectstatic"
 mtime "collectstatic.time" "${start}"
 
-# Create .profile script for application runtime environment variables.
-set_env PATH "\$HOME/.heroku/python/bin:\$PATH"
-set_env PYTHONUNBUFFERED true
-set_env PYTHONHOME "\$HOME/.heroku/python"
 
+# Progamatically create .profile.d script for application runtime environment variables.
+
+# Set the PATH to include Python / pip / pipenv / etc.
+set_env PATH "\$HOME/.heroku/python/bin:\$PATH"
+# Tell Python to run in unbuffered mode.
+set_env PYTHONUNBUFFERED true
+# Tell Python where it lives.
+set_env PYTHONHOME "\$HOME/.heroku/python"
+# Set variables for C libraries.
 set_env LIBRARY_PATH "\$HOME/.heroku/vendor/lib:\$HOME/.heroku/python/lib:\$LIBRARY_PATH"
 set_env LD_LIBRARY_PATH "\$HOME/.heroku/vendor/lib:\$HOME/.heroku/python/lib:\$LD_LIBRARY_PATH"
-
+# Locale.
 set_default_env LANG en_US.UTF-8
+# The Python hash seed is set to random.
 set_default_env PYTHONHASHSEED random
+# Tell Python to look for Python modules in the /app dir. Don't change this.
 set_default_env PYTHONPATH "\$HOME"
 
-# python expects to be in /app, if at runtime, it is not, set
+# Python expects to be in /app, if at runtime, it is not, set
-# up symlinks... this can occur when the subdir buildpack is used
+# up symlinks… this can occur when the subdir buildpack is used.
 cat <<EOT >> "$PROFILE_PATH"
 if [[ \$HOME != "/app" ]]; then
     mkdir -p /app/.heroku
@@ -244,8 +354,7 @@ EOT
 cp "$ROOT_DIR/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH"
 cp "$ROOT_DIR/vendor/python.gunicorn.sh" "$GUNICORN_PROFILE_PATH"
 
-
+# Experimental post_compile hook. Don't remove this.
-# Experimental post_compile hook.
 # shellcheck source=bin/steps/hooks/post_compile
 source "$BIN_DIR/steps/hooks/post_compile"
 
@@ -253,8 +362,7 @@ source "$BIN_DIR/steps/hooks/post_compile"
 # shellcheck source=bin/steps/eggpath-fix2
 source "$BIN_DIR/steps/eggpath-fix2"
 
-# Store new artifacts in cache.
+# Store new artifacts in the cache.
-
 rm -rf "$CACHE_DIR/.heroku/python"
 rm -rf "$CACHE_DIR/.heroku/python-version"
 rm -rf "$CACHE_DIR/.heroku/python-stack"

bin/steps/collectstatic

@@ -11,7 +11,7 @@
 #   - $DEBUG_COLLECTSTATIC: upon failure, print out environment variables.
 
 # shellcheck source=bin/utils
-source $BIN_DIR/utils
+source "$BIN_DIR/utils"
 
 # Location of 'manage.py', if it exists.
 MANAGE_FILE=$(find . -maxdepth 3 -type f -name 'manage.py' -printf '%d\t%P\n' | sort -nk1 | cut -f2 | head -1)
@@ -44,22 +44,20 @@ if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALL
     # Display a warning if collectstatic failed.
     [ "$COLLECTSTATIC_STATUS" -ne 0 ] && {
 
-        mcount "failure.collectstatic"
-
         if grep -q 'SyntaxError' "$COLLECTSTATIC_LOG"; then
             mcount "failure.collectstatic.syntax-error"
-        fi
 
-        if grep -q 'ImproperlyConfigured' "$COLLECTSTATIC_LOG"; then
+        elif grep -q 'ImproperlyConfigured' "$COLLECTSTATIC_LOG"; then
             mcount "failure.collectstatic.improper-configuration"
-        fi
 
-        if grep -q 'The CSS file' "$COLLECTSTATIC_LOG"; then
+        elif grep -q 'The CSS file' "$COLLECTSTATIC_LOG"; then
             mcount "failure.collectstatic.fancy-references"
-        fi
 
-        if grep -q 'OSError' "$COLLECTSTATIC_LOG"; then
+        elif grep -q 'OSError' "$COLLECTSTATIC_LOG"; then
             mcount "failure.collectstatic.missing-file"
+
+        else
+            mcount "failure.collectstatic.other"
         fi
 
         echo

bin/steps/pip-install

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # shellcheck source=bin/utils
-source $BIN_DIR/utils
+source "$BIN_DIR/utils"
 
 if [ ! "$SKIP_PIP_INSTALL" ]; then
 
@@ -20,9 +20,6 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     mcount "tool.pip"
 
     # Count expected build failures.
-    if grep -q 'wsgiref' requirements.txt; then
-        mcount "failure.wsgiref"
-    fi
     if grep -q '==0.0.0' requirements.txt; then
         mcount "failure.none-version"
     fi
@@ -35,10 +32,10 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     show-warnings
 
     if [[ ! $PIP_STATUS -eq 0 ]]; then
+        mcount "failure.pip-install"
         exit 1
     fi
 
-
     # Smart Requirements handling
     cp requirements.txt .heroku/python/requirements-declared.txt
     /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt

bin/steps/pip-uninstall

@@ -3,7 +3,7 @@
 set +e
 # Install dependencies with Pip.
 # shellcheck source=bin/utils
-source $BIN_DIR/utils
+source "$BIN_DIR/utils"
 
 if [ ! "$SKIP_PIP_INSTALL" ]; then
 
@@ -20,7 +20,7 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
 
     if [[ -s .heroku/python/requirements-stale.txt ]]; then
       puts-step "Uninstalling stale dependencies"
-      /app/.heroku/python/bin/pip uninstall -r .heroku/python/requirements-stale.txt -y --exists-action=w | cleanup | indent
+      /app/.heroku/python/bin/pip uninstall -r .heroku/python/requirements-stale.txt -y --exists-action=w --disable-pip-version-check | cleanup | indent
     fi
   fi
 

bin/steps/pipenv

@@ -3,7 +3,7 @@
 # export CLINT_FORCE_COLOR=1
 # export PIPENV_FORCE_COLOR=1
 # shellcheck source=bin/utils
-source $BIN_DIR/utils
+source "$BIN_DIR/utils"
 set -e
 
 if [[ -f Pipfile.lock ]]; then
@@ -41,7 +41,7 @@ if [ ! "$SKIP_PIPENV_INSTALL" ]; then
             export PIP_EXTRA_INDEX_URL
         fi
 
-        export PIPENV_VERSION="11.8.2"
+        export PIPENV_VERSION="2018.5.18"
 
         # Install pipenv.
         /app/.heroku/python/bin/pip install pipenv==$PIPENV_VERSION --upgrade &> /dev/null

bin/steps/pipenv-python-version

@@ -22,7 +22,10 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
                     echo "$LATEST_2" > "$BUILD_DIR/runtime.txt"
                 fi
                 if [ "$PYTHON" = 3.6 ]; then
-                    echo "$LATEST_3" > "$BUILD_DIR/runtime.txt"
+                    echo "$LATEST_36" > "$BUILD_DIR/runtime.txt"
+                fi
+                if [ "$PYTHON" = 3.7 ]; then
+                    echo "$LATEST_37" > "$BUILD_DIR/runtime.txt"
                 fi
             fi
 

bin/steps/pylibmc

@@ -9,7 +9,7 @@
 #
 # This script is invoked by [`bin/compile`](/).
 
-if [[ "$STACK" == "heroku-16" ]]; then
+if [[ "$STACK" != "cedar-14" ]]; then
     # libmemcached is pre-installed in the stack image so there is no need to vendor it.
     return 0
 fi

bin/steps/python

@@ -12,19 +12,49 @@ if [[ $PYTHON_VERSION =~ ^python-2 ]]; then
     puts-warn "The latest version of Python 2 is $LATEST_2 (you are using $PYTHON_VERSION, which is unsupported)."
     puts-warn "We recommend upgrading by specifying the latest version ($LATEST_2)."
     echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+  else
+    echo "       Using supported version of Python 2 ($PYTHON_VERSION)"
   fi
 else
-  if [[ "$PYTHON_VERSION" != "$LATEST_3" ]]; then
+  if [[ $PYTHON_VERSION =~ ^python-3 ]]; then
-    puts-warn "The latest version of Python 3 is $LATEST_3 (you are using $PYTHON_VERSION, which is unsupported)."
+    if [[ $PYTHON_VERSION =~ ^python-3.7 ]]; then
-    puts-warn "We recommend upgrading by specifying the latest version ($LATEST_3)."
+      if [[ "$PYTHON_VERSION" != "$LATEST_37" ]]; then
-    echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+      puts-warn "The latest version of Python 3.7 is $LATEST_37 (you are using $PYTHON_VERSION, which is unsupported)."
+      puts-warn "We recommend upgrading by specifying the latest version ($LATEST_37)."
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+      else
+        echo "       Using supported version of Python 3.7 ($PYTHON_VERSION)"
+      fi
+    else
+      if [[ $PYTHON_VERSION =~ ^python-3.6 ]]; then
+        if [[ "$PYTHON_VERSION" != "$LATEST_36" ]]; then
+          puts-warn "The latest version of Python 3.6 is $LATEST_36 (you are using $PYTHON_VERSION, which is unsupported)."
+          puts-warn "We recommend upgrading by specifying the latest version ($LATEST_36)."
+          echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+        else
+          echo "       Using supported version of Python 3.6 ($PYTHON_VERSION)"
+        fi
+      else
+        puts-warn "Heroku supports runtime versions $LATEST_37, $LATEST_36 and $LATEST_2." 
+        puts-warn "You are using $PYTHON_VERSION, which is unsupported."
+        puts-warn "We recommend upgrading by specifying the default supported version ($LATEST_36)."
+        echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+      fi
+    fi
   fi
-
 fi
 
 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
     puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache"
-    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
+    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
+fi
+
+# need to clear the cache for first time installing SQLite3,
+# since the version is changing and could lead to runtime errors
+# with compiled extensions.
+if [ -d .heroku/python ] && [ ! -f .heroku/python-sqlite3-version ] && python_sqlite3_check "$PYTHON_VERSION"; then
+  puts-step "Need to update SQLite3, clearing cache"
+  rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
 fi
 
 if [ -f .heroku/python-version ]; then
@@ -36,6 +66,7 @@ if [ -f .heroku/python-version ]; then
   fi
 fi
 
+
 if [ ! "$SKIP_INSTALL" ]; then
     puts-step "Installing $PYTHON_VERSION"
 
@@ -58,6 +89,7 @@ if [ ! "$SKIP_INSTALL" ]; then
   hash -r
 fi
 
+
 # If Pip isn't up to date:
 if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
 
@@ -67,7 +99,8 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
   rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
   rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
 
-  /app/.heroku/python/bin/python "$ROOT_DIR/vendor/get-pip.py" &> /dev/null
+  /app/.heroku/python/bin/python "$ROOT_DIR/vendor/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null
+  /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
 
 fi
 

bin/steps/sqlite3

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
+
+sqlite3_version() {
+  SQLITE3_VERSION=${SQLITE3_VERSION:-$(dpkg -s libsqlite3-0 | grep Version | sed 's/Version: //')}
+
+  export SQLITE3_VERSION
+}
+
+sqlite3_install() {
+  HEROKU_PYTHON_DIR="$1"
+  SQLITE3_VERSION="$2"
+  HEADERS_ONLY="$3"
+
+  mkdir -p "$HEROKU_PYTHON_DIR"
+
+  APT_CACHE_DIR="$HEROKU_PYTHON_DIR/apt/cache"
+  APT_STATE_DIR="$HEROKU_PYTHON_DIR/apt/state"
+
+  mkdir -p "$APT_CACHE_DIR/archives/partial"
+  mkdir -p "$APT_STATE_DIR/lists/partial"
+
+  APT_OPTIONS="-o debug::nolocking=true"
+  APT_OPTIONS="$APT_OPTIONS -o dir::cache=$APT_CACHE_DIR"
+  APT_OPTIONS="$APT_OPTIONS -o dir::state=$APT_STATE_DIR"
+  APT_OPTIONS="$APT_OPTIONS -o dir::etc::sourcelist=/etc/apt/sources.list"
+
+  apt-get $APT_OPTIONS update > /dev/null 2>&1
+  if [ -z "$HEADERS_ONLY" ]; then
+    apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION" sqlite3="$SQLITE3_VERSION" > /dev/null 2>&1
+  else
+    apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION"
+  fi
+
+  find "$APT_CACHE_DIR/archives/" -name "*.deb" -exec dpkg -x {} "$HEROKU_PYTHON_DIR/sqlite3/" \;
+
+  mkdir -p "$HEROKU_PYTHON_DIR/include"
+  mkdir -p "$HEROKU_PYTHON_DIR/lib"
+
+  # remove old sqlite3 libraries/binaries
+  find "$HEROKU_PYTHON_DIR/include/" -name "sqlite3*.h" -exec rm -f {} \;
+  find "$HEROKU_PYTHON_DIR/lib/" -name "libsqlite3.*" -exec rm -f {} \;
+  rm -f "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc"
+  rm -f "$HEROKU_PYTHON_DIR/bin/sqlite3"
+
+  # copy over sqlite3 headers & bins and setup linking against the stack image library
+  mv "$HEROKU_PYTHON_DIR/sqlite3/usr/include/"* "$HEROKU_PYTHON_DIR/include/"
+  mv "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu"/libsqlite3.*a "$HEROKU_PYTHON_DIR/lib/"
+  mkdir -p "$HEROKU_PYTHON_DIR/lib/pkgconfig"
+  # set the right prefix/lib directories
+  sed -e 's/prefix=\/usr/prefix=\/app\/.heroku\/python/' -e 's/\/x86_64-linux-gnu//' "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/pkgconfig/sqlite3.pc" > "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc"
+  # need to point the libsqlite3.so to the stack image library for /usr/bin/ld -lsqlite3
+  SQLITE3_LIBFILE="/usr/lib/x86_64-linux-gnu/$(readlink -n "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/libsqlite3.so")"
+  ln -s "$SQLITE3_LIBFILE" "$HEROKU_PYTHON_DIR/lib/libsqlite3.so"
+  if [ -z "$HEADERS_ONLY" ]; then
+    mv "$HEROKU_PYTHON_DIR/sqlite3/usr/bin"/* "$HEROKU_PYTHON_DIR/bin/"
+  fi
+
+  # cleanup
+  rm -rf "$HEROKU_PYTHON_DIR/sqlite3/"
+  rm -rf "$HEROKU_PYTHON_DIR/apt/"
+}
+
+buildpack_sqlite3_install() {
+  sqlite3_version
+  HEROKU_PYTHON_DIR="$BUILD_DIR/.heroku/python"
+
+  SQLITE3_VERSION_FILE="$BUILD_DIR/.heroku/python-sqlite3-version"
+  if [ -f "$SQLITE3_VERSION_FILE" ]; then
+    INSTALLED_SQLITE3_VERSION=$(cat "$SQLITE3_VERSION_FILE")
+  fi
+
+  # python version check
+  if python_sqlite3_check "$PYTHON_VERSION"; then
+    # only install if the sqlite3 version has changed
+    if [ "$INSTALLED_SQLITE3_VERSION" != "$SQLITE3_VERSION" ]; then
+      puts-step "Installing SQLite3"
+      sqlite3_install "$BUILD_DIR/.heroku/python" "$SQLITE3_VERSION"
+
+      # save version installed
+      mkdir -p "$CACHE_DIR/.heroku/"
+      echo "$SQLITE3_VERSION" > "$CACHE_DIR/.heroku/python-sqlite3-version"
+    fi
+  fi
+}

bin/utils

@@ -58,3 +58,41 @@ measure-size() {
   echo "$(du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')"
 }
 
+# Python version operator >
+version_gt() {
+  test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1";
+}
+
+# Python verison operator >=
+version_gte() {
+  if [ "$1" == "$2" ]; then
+    return 0
+  fi
+
+  version_gt "$1" "$2"
+}
+
+# Check if Python 2
+python2_check() {
+  VERSION="$1"
+
+  version_gte "$VERSION" "python-2.7.0" && version_gt "python-3.0.0" "$VERSION"
+}
+
+# Check if Python 3
+python3_check() {
+  VERSION="$1"
+
+  version_gte "$VERSION" "python-3.0.0" && version_gt "python-4.0.0" "$VERSION"
+}
+
+# Check if Python version needs to install SQLite3
+python_sqlite3_check() {
+  VERSION="$1"
+  MIN_PYTHON_3="python-3.6.6"
+  MIN_PYTHON_2="python-2.7.15"
+
+  ( python2_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_2" ) \
+    || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" ) \
+    || ( version_gte "$VERSION" "3.7.0" )
+}

buildpack.toml

@@ -0,0 +1,13 @@
+[buildpack]
+name = "Python"
+
+  [publish.Ignore]
+  files = [
+    "test/",
+    ".gitignore",
+    ".dockerignore",
+    ".github/",
+    "Dockerfile",
+    "Pipfile",
+    "Pipfile.lock"
+  ]

builds/runtimes/python-2.7.15

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/2.7.15/Python-2.7.15.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-2.7.15 src
+cd src
+
+./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +

builds/runtimes/python-3.6.5

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.6.5/Python-3.6.5.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.6.5 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python

builds/runtimes/python-3.6.6

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.6.6/Python-3.6.6.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.6.6 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python

builds/runtimes/python-3.7.0

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.7.0/Python-3.7.0.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.7.0 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python

etc/ci-setup.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+sudo apt-get -qq update
+sudo apt-get install software-properties-common
+curl --fail --retry 3 --retry-delay 1 --connect-timeout 3 --max-time 30 https://cli-assets.heroku.com/install-ubuntu.sh | sh

etc/publish.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+
+BP_NAME=${1:-"heroku/python"}
+
+curVersion=$(heroku buildpacks:versions "$BP_NAME" | awk 'FNR == 3 { print $1 }')
+newVersion="v$((curVersion + 1))"
+
+read -p "Deploy as version: $newVersion [y/n]? " choice
+case "$choice" in
+  y|Y ) echo "";;
+  n|N ) exit 0;;
+  * ) exit 1;;
+esac
+
+originMaster=$(git rev-parse origin/master)
+echo "Tagging commit $originMaster with $newVersion... "
+git tag "$newVersion" "${originMaster:?}"
+git push origin refs/tags/$newVersion
+
+heroku buildpacks:publish "$BP_NAME" "$newVersion"
+
+if [ $(git tag | grep -q previous-version) ]; then
+    echo "Updating previous-version tag"
+    git tag -d previous-version
+    git push origin :previous-version
+    git tag previous-version latest-version
+fi
+if [ $(git tag | grep -q latest-version) ]; then
+    echo "Updating latest-version tag"
+    git tag -d latest-version
+    git push origin :latest-version
+    git tag latest-version "${originMaster:?}"
+    git push --tags
+fi
+
+echo "Done."

hatchet.json

@@ -0,0 +1,5 @@
+{
+  "python": [
+    "heroku/python-getting-started"
+  ]
+}

hatchet.lock

@@ -0,0 +1,3 @@
+---
+- - "./repos/python/python-getting-started"
+  - f56b90499ec11e1c9576da2f8c7331300e189db3

spec/hatchet/python_spec.rb

@@ -0,0 +1,10 @@
+require_relative '../spec_helper'
+
+describe "Python!!!!!!!!!!!" do
+  it "🐍" do
+    Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app|
+      expect(app.output).to           match(/Installing pip/)
+      expect(app.run('python -V')).to match(/3.6.6/)
+    end
+  end
+end

spec/spec_helper.rb

@@ -0,0 +1,23 @@
+ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/heroku/heroku-buildpack-python.git'
+
+require 'rspec/core'
+require 'rspec/retry'
+require 'hatchet'
+
+require 'date'
+
+RSpec.configure do |config|
+  config.full_backtrace      = true
+  config.verbose_retry       = true # show retry status in spec process
+  config.default_retry_count = 2 if ENV['IS_RUNNING_ON_CI'] # retry all tests that fail again
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end
+
+if ENV['TRAVIS']
+  # Don't execute tests against "merge" commits
+  exit 0 if ENV['TRAVIS_PULL_REQUEST'] != 'false' && ENV['TRAVIS_BRANCH'] == 'master'
+end
+
+DEFAULT_STACK = 'heroku-16'

test/fixtures/git-egg/requirements.txt

@@ -0,0 +1 @@
+-e git+https://github.com/requests/requests.git#egg=requests

test/fixtures/nltk/runtime.txt

@@ -1 +1 @@
-python-2.7.13
+python-3.6.6

test/fixtures/pipenv-lock/Pipfile.lock

@@ -1,27 +1,14 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "397f2c55e3558ea57d292e3fc19b34e483770e5ec02cdedfb1f330680cd26635"
+            "sha256": "09ad9dcae1870ba083f43c5a05ed8943b23bd4c27e61a13ecf4e16d18500ad98"
-        },
-        "host-environment-markers": {
-            "implementation_name": "cpython",
-            "implementation_version": "3.6.3",
-            "os_name": "posix",
-            "platform_machine": "x86_64",
-            "platform_python_implementation": "CPython",
-            "platform_release": "16.7.0",
-            "platform_system": "Darwin",
-            "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64",
-            "python_full_version": "3.6.3",
-            "python_version": "3.6",
-            "sys_platform": "darwin"
         },
         "pipfile-spec": 6,
         "requires": {},
         "sources": [
             {
                 "name": "pypi",
-                "url": "https://pypi.python.org/simple",
+                "url": "https://pypi.org/simple",
                 "verify_ssl": true
             }
         ]
@@ -29,22 +16,23 @@
     "default": {
         "delegator.py": {
             "hashes": [
-                "sha256:2575c4adc923ad0b8fdaa433f862b2b7cf21982717fb23cc895fd8f249ea820c",
+                "sha256:2d46966a7f484d271b09e2646eae1e9acadc4fdf2cb760c142f073e81c927d8d",
-                "sha256:495e11ada66648650171a6c9a188df4eb050b235abff8771f41ee8a064eb9ded"
+                "sha256:58f3ea6fe36680e1d828e2e66e52844b826f186409dfee4436e42351b0e699fe"
             ],
-            "version": "==0.0.13"
+            "index": "pypi",
+            "version": "==0.1.0"
         },
         "pexpect": {
             "hashes": [
-                "sha256:f853b52afaf3b064d29854771e2db509ef80392509bde2dd7a6ecf2dfc3f0018",
+                "sha256:9783f4644a3ef8528a6f20374eeb434431a650c797ca6d8df0d81e30fffdfa24",
-                "sha256:3d132465a75b57aa818341c6521392a06cc660feb3988d7f1074f39bd23c9a92"
+                "sha256:9f8eb3277716a01faafaba553d629d3d60a1a624c7cf45daa600d2148c30020c"
             ],
-            "version": "==4.2.1"
+            "version": "==4.5.0"
         },
         "ptyprocess": {
             "hashes": [
-                "sha256:e8c43b5eee76b2083a9badde89fd1bbce6c8942d1045146e100b7b5e014f4f1a",
+                "sha256:e64193f0047ad603b71f202332ab5527c5e52aa7c8b609704fc28c0dc20c4365",
-                "sha256:e64193f0047ad603b71f202332ab5527c5e52aa7c8b609704fc28c0dc20c4365"
+                "sha256:e8c43b5eee76b2083a9badde89fd1bbce6c8942d1045146e100b7b5e014f4f1a"
             ],
             "version": "==0.5.2"
         }

test/fixtures/python2/runtime.txt

@@ -1 +1 @@
-python-2.7.14
+python-2.7.15

test/run

@@ -12,7 +12,7 @@ testPipenvLock() {
 
 testPipenvVersion() {
   compile "pipenv-version"
-  assertCaptured "3.6.4"
+  assertCaptured "3.6.6"
   assertCapturedSuccess
 }
 
@@ -39,6 +39,11 @@ testGEOS() {
 }
 
 testNLTK() {
+  # NOTE: This is a RuntimeWarning emitted by Python 3's runpy.py script
+  # which is what is used when you call `python -m <module>`. This is due to
+  # how nltk imports things. It's not actually an error, but it would probably
+  # be bad to silence in Production.
+  export PYTHONWARNINGS="ignore::RuntimeWarning"
   compile "nltk"
   assertCaptured "Downloading NLTK packages: city_database stopwords"
   assertCapturedSuccess
@@ -76,14 +81,20 @@ testPylibmc() {
 }
 
 testPython2() {
-  compile "python2"
+    compile "python2"
-  assertCaptured "python-2.7.14"
+    assertCaptured "python-2.7.15"
-  assertCapturedSuccess
+    assertCapturedSuccess
 }
 
 testPython3() {
   compile "python3"
-  assertCaptured "python-3.6.4"
+  assertCaptured "python-3.6.6"
+  assertCapturedSuccess
+}
+
+testGitEgg() {
+  compile "git-egg"
+  assertCaptured "requests"
   assertCapturedSuccess
 }
 

tests.sh

@@ -14,3 +14,8 @@ if [[ "$STACK" == "heroku-16" ]]; then
     make test-heroku-16
     exit $?
 fi
+
+if [[ "$STACK" == "heroku-18" ]]; then
+    make test-heroku-18
+    exit $?
+fi

vendor/pipenv-to-pip

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import io
 import json
 import sys
 
@@ -7,7 +8,7 @@ import sys
 def main():
     INFILE = sys.argv[1]
 
-    with open(INFILE, 'rb') as f:
+    with io.open(INFILE, 'r', encoding='utf-8') as f:
         lockfile = json.load(f)
 
     packages = []