Release v185 (#1114)

Closes @W-8381914@.

.travis.yml

@@ -23,10 +23,6 @@ jobs:
 
 env:
   jobs:
-    - STACK=cedar-14 TEST_CMD=test/run-deps
-    - STACK=cedar-14 TEST_CMD=test/run-versions
-    - STACK=cedar-14 TEST_CMD=test/run-features
-
     - STACK=heroku-16 TEST_CMD=test/run-deps
     - STACK=heroku-16 TEST_CMD=test/run-versions
     - STACK=heroku-16 TEST_CMD=test/run-features
@@ -34,6 +30,10 @@ env:
     - STACK=heroku-18 TEST_CMD=test/run-deps
     - STACK=heroku-18 TEST_CMD=test/run-versions
     - STACK=heroku-18 TEST_CMD=test/run-features
+
+    - STACK=heroku-20 TEST_CMD=test/run-deps
+    - STACK=heroku-20 TEST_CMD=test/run-versions
+    - STACK=heroku-20 TEST_CMD=test/run-features
   global:
     - HATCHET_RETRIES=3
     - IS_RUNNING_ON_CI=true

CHANGELOG.md

@@ -3,6 +3,30 @@
 ## Unreleased
 
 
+## v185 (2020-11-12)
+
+- Error if the unsupported `BUILD_WITH_GEO_LIBRARIES` env var is set (#1115).
+- Remove deprecated GDAL/GEOS/PROJ support (#1113).
+- Remove vendored `jq` binary (#1112).
+- Remove redundant Mercurial install step (#1111).
+- Remove support for the Cedar-14 stack (#1110).
+
+## v184 (2020-10-21)
+
+- Vendor buildpack-stdlib instead of fetching from S3 (#1100).
+- Fix metric names for metrics emitted within `sub_env` (#1099).
+
+## v183 (2020-10-12)
+
+- Add support for Heroku-20 (#968).
+
+## v182 (2020-10-06)
+
+- Python 3.9.0 is now available (CPython) (#1090).
+- Migrate from the `lang-python` S3 bucket to `heroku-buildpack-python` (#1089).
+- Remove `vendor/shunit2` (#1086).
+- Replace `BUILDPACK_VENDOR_URL` and `USE_STAGING_BINARIES` with `BUILDPACK_S3_BASE_URL` (#1085).
+
 ## v181 (2020-09-29)
 
 - PyPy 2.7 and 3.6, version 7.3.2 are now available (Note: PyPy support is in beta) (#1081).

Gemfile.lock

@@ -3,17 +3,17 @@ GEM
   specs:
     diff-lcs (1.4.4)
     erubis (2.7.0)
-    excon (0.76.0)
+    excon (0.78.0)
     heroics (0.1.1)
       erubis (~> 2.0)
       excon
       moneta
       multi_json (>= 1.9.2)
-    heroku_hatchet (7.3.0)
+    heroku_hatchet (7.3.3)
       excon (~> 0)
       platform-api (~> 3)
       rrrretry (~> 1)
-      thor (~> 0)
+      thor (~> 1)
       threaded (~> 0)
     moneta (1.0.0)
     multi_json (1.15.0)
@@ -28,22 +28,22 @@ GEM
     rake (13.0.1)
     rate_throttle_client (0.1.2)
     rrrretry (1.0.0)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.0)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
+      rspec-support (~> 3.10.0)
     rspec-retry (0.6.2)
       rspec-core (> 3.3)
-    rspec-support (3.9.3)
-    thor (0.20.3)
+    rspec-support (3.10.0)
+    thor (1.0.1)
     threaded (0.0.4)
 
 PLATFORMS

Makefile

@@ -1,55 +1,58 @@
 # These targets are not files
-.PHONY: check test buildenv-heroku-16 buildenv-heroku-18 tools
+.PHONY: check test builder-image buildenv deploy-runtimes tools
 
 STACK ?= heroku-18
+STACKS ?= heroku-16 heroku-18 heroku-20
 TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
+ENV_FILE ?= builds/dockerenv.default
+BUILDER_IMAGE_PREFIX := heroku-python-build
 
-ifeq ($(STACK),cedar-14)
-	# Cedar-14 doesn't have a build image varient.
-	IMAGE_TAG := heroku/cedar:14
-else
-	# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
-	IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
-endif
+# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
+STACK_IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
 
 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
-	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
+	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/python
 	@shellcheck -x bin/steps/hooks/*
 
 test:
 	@echo "Running tests using: STACK=$(STACK) TEST_CMD='$(TEST_CMD)'"
-	@echo ""
-	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
-	@echo ""
-
-buildenv-heroku-16:
-	@echo "Creating build environment (heroku-16)..."
 	@echo
-	@docker build --pull -f $(shell pwd)/builds/heroku-16.Dockerfile -t python-buildenv-heroku-16 .
+	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(STACK_IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
+	@echo
+
+builder-image:
+	@echo "Generating binary builder image for $(STACK)..."
+	@echo
+	@docker build --pull -f builds/$(STACK).Dockerfile -t "$(BUILDER_IMAGE_PREFIX)-$(STACK)" .
+	@echo
+
+buildenv: builder-image
+	@echo "Starting build environment for $(STACK)..."
 	@echo
 	@echo "Usage..."
 	@echo
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ bob build runtimes/python-2.7.13"
-	@echo "  $$ bob deploy runtimes/python-2.7.13"
+	@echo "  $$ bob build runtimes/python-X.Y.Z"
 	@echo
-	@docker run -it --rm python-buildenv-heroku-16
+	@docker run --rm -it --env-file="$(ENV_FILE)" -v $(PWD)/builds:/app/builds "$(BUILDER_IMAGE_PREFIX)-$(STACK)" bash
 
-buildenv-heroku-18:
-	@echo "Creating build environment (heroku-18)..."
+deploy-runtimes:
+ifndef RUNTIMES
+	$(error No runtimes specified! Use: "make deploy-runtimes RUNTIMES='python-X.Y.Z ...' [STACKS='heroku-18 ...'] [ENV_FILE=...]")
+endif
+	@echo "Using: RUNTIMES='$(RUNTIMES)' STACKS='$(STACKS)' ENV_FILE='$(ENV_FILE)'"
 	@echo
-	@docker build --pull -f $(shell pwd)/builds/heroku-18.Dockerfile -t python-buildenv-heroku-18 .
-	@echo
-	@echo "Usage..."
-	@echo
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ bob build runtimes/python-2.7.13"
-	@echo "  $$ bob deploy runtimes/python-2.7.13"
-	@echo
-	@docker run -it --rm python-buildenv-heroku-18
+	@set -eu; for stack in $(STACKS); do \
+		$(MAKE) builder-image STACK=$${stack}; \
+		for runtime in $(RUNTIMES); do \
+			echo "Generating/deploying $${runtime} for $${stack}..."; \
+			echo; \
+			docker run --rm -it --env-file="$(ENV_FILE)" "$(BUILDER_IMAGE_PREFIX)-$${stack}" bob deploy "runtimes/$${runtime}"; \
+			echo; \
+		done; \
+	done
 
 tools:
 	git clone https://github.com/kennethreitz/pip-pop.git
 	mv pip-pop/bin/* vendor/pip-pop/
-	rm -fr pip-pop
+	rm -rf pip-pop

NOTICE

@@ -1,27 +1,5 @@
 This buildpack includes some vendorized packages to ease installation.
 
-jq license
-----------
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-
 shunit2 license
 ---------------
 

README.md

@@ -62,6 +62,7 @@ Specify a Python Runtime
 
 Supported runtime options include:
 
+- `python-3.9.0`
 - `python-3.8.6`
 - `python-3.7.9`
 - `python-3.6.12`

bin/compile

@@ -15,10 +15,8 @@
 # Fail fast and fail hard.
 set -eo pipefail
 
-# Boostrap the Buildpack Standard Library.
-# Disable unused env var warning since shellcheck doesn't know about the stdlib.
-# shellcheck disable=2034
-BPLOG_PREFIX="buildpack.python"
+# Used by buildpack-stdlib's metrics features.
+export BPLOG_PREFIX="buildpack.python"
 export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
 
 [ "$BUILDPACK_XTRACE" ] && set -o xtrace
@@ -37,23 +35,19 @@ ENV_DIR=$3
 # Export Path variables, for use in sub-scripts.
 export BUILD_DIR CACHE_DIR ENV_DIR
 
-# Set the Buildpack's internet target for downloading Python distributions.
-# The user can provide BUILDPACK_VENDOR_URL to specify a custom target.
+# Set the base URL for downloading buildpack assets like Python runtimes.
+# The user can provide BUILDPACK_S3_BASE_URL to specify a custom target.
 # Note: this is designed for non-Heroku use, as it does not use the user-provided
 # environment variable mechanism (the ENV_DIR).
-VENDOR_URL="https://lang-python.s3.amazonaws.com/$STACK"
-if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then
-    VENDOR_URL="$BUILDPACK_VENDOR_URL"
-elif [[ -n ${USE_STAGING_BINARIES} ]]; then
-    VENDOR_URL="$USE_STAGING_BINARIES/$STACK"
-fi
-export VENDOR_URL
+S3_BASE_URL="${BUILDPACK_S3_BASE_URL:-"https://heroku-buildpack-python.s3.amazonaws.com"}"
+# This has to be exported since it's used by the geo-libs step which is run in a subshell.
 
 # Default Python Versions
 # shellcheck source=bin/default_pythons
 source "$BIN_DIR/default_pythons"
 
 # Supported Python Branches
+PY39="python-3.9"
 PY38="python-3.8"
 PY37="python-3.7"
 PY36="python-3.6"
@@ -64,7 +58,8 @@ PYPY27="pypy2.7"
 PYPY36="pypy3.6"
 
 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
-DEFAULT_PYTHON_STACK="cedar-14"
+# TODO: Remove this and require that STACK be set explicitly.
+DEFAULT_PYTHON_STACK="heroku-18"
 
 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.
@@ -101,6 +96,31 @@ source "$BIN_DIR/utils"
 # shellcheck source=bin/warnings
 source "$BIN_DIR/warnings"
 
+if [[ "${STACK}" == "cedar-14" ]]; then
+  mcount "failure.unsupported.cedar-14"
+  puts-warn "The Cedar-14 stack is no longer supported by the latest release of this buildpack."
+  puts-warn
+  puts-warn "Please switch to the Cedar-14 support branch by using this buildpack URL:"
+  puts-warn "https://github.com/heroku/heroku-buildpack-python#cedar-14"
+  puts-warn
+  puts-warn "For instructions on how to change the buildpacks used by an app, see:"
+  puts-warn "https://devcenter.heroku.com/articles/buildpacks#setting-a-buildpack-on-an-application"
+  exit 1
+fi
+
+if [[ -f "${ENV_DIR}/BUILD_WITH_GEO_LIBRARIES" ]]; then
+  mcount "failure.unsupported.BUILD_WITH_GEO_LIBRARIES"
+  puts-warn "The Python buildpack's BUILD_WITH_GEO_LIBRARIES functonality is no longer supported:"
+  puts-warn "https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-now-deprecated"
+  puts-warn
+  puts-warn "For GDAL, GEOS and PROJ support, use the Geo buildpack alongside the Python buildpack:"
+  puts-warn "https://github.com/heroku/heroku-geo-buildpack"
+  puts-warn
+  puts-warn "To remove this error message, unset the BUILD_WITH_GEO_LIBRARIES variable using:"
+  puts-warn "heroku config:unset BUILD_WITH_GEO_LIBRARIES"
+  exit 1
+fi
+
 # Make the directory in which we will create symlinks from the temporary build directory
 # to `/app`.
 # Symlinks are required, since Python is not a portable installation.
@@ -129,8 +149,9 @@ export PYTHONUNBUFFERED=1
 # Set the locale to a well-known and expected standard.
 export LANG=en_US.UTF-8
 # `~/.heroku/vendor` is an place where the buildpack may stick pre-build binaries for known
-# C dependencies (e.g. libmemcached on cedar-14). This section configures Python (GCC, more specifically)
+# C dependencies. This section configures Python (GCC, more specifically)
 # and pip to automatically include these paths when building binaries.
+# TODO: Stop adding .heroku/vendor here now that the buildpack no longer vendors anything.
 export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
 export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
 export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
@@ -170,8 +191,6 @@ cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
 cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
 # A plain text file which contains the current sqlite3 version being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-sqlite3-version" .heroku/ &> /dev/null || true
-# Any pre-compiled binaries, provided by the buildpack.
-cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true
 # "editable" installations of code repositories, via pip or pipenv.
 if [[ -d "$CACHE_DIR/.heroku/src" ]]; then
   cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true
@@ -257,27 +276,6 @@ fi
 # shellcheck source=bin/steps/eggpath-fix
 source "$BIN_DIR/steps/eggpath-fix"
 
-# Mercurial support.
-# If a customer appears to be using mercurial for dependency resolution, we install it first.
-# Note: this only applies to pip, not pipenv. This can likely be removed, over time. Measure it first.
-# shellcheck source=bin/steps/mercurial
-source "$BIN_DIR/steps/mercurial"
-
-# Pylibmc support.
-# On cedar-14, libmemcached was not available. The buildpack provides its own version, instead.
-# shellcheck source=bin/steps/pylibmc
-source "$BIN_DIR/steps/pylibmc"
-
-# Support for Geo libraries. This is deprecated functionality, only functional on cedar-14.
-# It is undocumented.
-# shellcheck source=bin/steps/geo-libs
-sub_env "$BIN_DIR/steps/geo-libs"
-
-# GDAL support.
-# This is part of the Geo support.
-# shellcheck source=bin/steps/gdal
-source "$BIN_DIR/steps/gdal"
-
 # SQLite3 support.
 # This sets up and installs sqlite3 dev headers and the sqlite3 binary but not the
 # libsqlite3-0 library since that exists on the stack image.
@@ -309,7 +307,7 @@ mtime "nltk.download.time" "${start}"
 # and copying it into the proper place (the logical place to do this was early, but it must be done here).
 # In CI, $BUILD_DIR is /app.
 if [[ ! "$BUILD_DIR" == "/app" ]]; then
-  rm -fr "$BUILD_DIR/.heroku/src"
+  rm -rf "$BUILD_DIR/.heroku/src"
   deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src"
 fi
 
@@ -375,7 +373,6 @@ mkdir -p "$CACHE_DIR/.heroku"
 cp -R .heroku/python "$CACHE_DIR/.heroku/"
 cp -R .heroku/python-version "$CACHE_DIR/.heroku/"
 cp -R .heroku/python-stack "$CACHE_DIR/.heroku/" &> /dev/null || true
-cp -R .heroku/vendor "$CACHE_DIR/.heroku/" &> /dev/null || true
 if [[ -d .heroku/src ]]; then
   cp -R .heroku/src "$CACHE_DIR/.heroku/" &> /dev/null || true
 fi

bin/default_pythons

@@ -6,6 +6,7 @@
 # shellcheck disable=2034
 
 DEFAULT_PYTHON_VERSION="python-3.6.12"
+LATEST_39="python-3.9.0"
 LATEST_38="python-3.8.6"
 LATEST_37="python-3.7.9"
 LATEST_36="python-3.6.12"

bin/steps/gdal

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-# This script serves as the GDAL build step of the
-# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
-# compiler.
-#
-# A [buildpack](https://devcenter.heroku.com/articles/buildpacks) is an
-# adapter between a Python application and Heroku's runtime.
-#
-# This script is invoked by [`bin/compile`](/).
-
-# The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
-
-PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
-
-# Syntax sugar.
-# shellcheck source=bin/utils
-source "$BIN_DIR/utils"
-
-# If GDAL exists within requirements, use vendored gdal.
-if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
-
-  if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then
-
-    puts-warn "The vendored GDAL package in the Heroku Python Buildpack now deprecated."
-    puts-warn "To enable GDAL use an alternative buildpack is available here - https://github.com/heroku/heroku-geo-buildpack"
-
-    echo "-----> Noticed GDAL. Bootstrapping gdal."
-    mkdir -p .heroku/vendor
-    # Download and extract cryptography into target vendor directory.
-    curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
-    mcount "steps.vendor.gdal"
-  fi
-
-  GDAL=$(pwd)/vendor
-  export GDAL
-fi

bin/steps/geo-libs

@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-
-# This script serves as the GDAL build step of the
-# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
-# compiler.
-#
-# A [buildpack](https://devcenter.heroku.com/articles/buildpacks) is an
-# adapter between a Python application and Heroku's runtime.
-#
-# This script is invoked by [`bin/compile`](/).
-
-# The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
-VENDORED_GEOS="${VENDOR_URL}/libraries/vendor/geos.tar.gz"
-VENDORED_PROJ="${VENDOR_URL}/libraries/vendor/proj.tar.gz"
-
-PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
-
-# Syntax sugar.
-# shellcheck source=bin/utils
-source "$BIN_DIR/utils"
-
-# If GDAL exists within requirements, use vendored gdal.
-if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
-  mcount "buildvar.BUILD_WITH_GEO_LIBRARIES"
-
-  puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."
-  puts-warn "An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
-
-  if [ ! -f ".heroku/vendor/bin/proj" ]; then
-    echo "-----> Bootstrapping gdal, geos, proj."
-    mkdir -p .heroku/vendor
-    # Download and extract cryptography into target vendor directory.
-    curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
-    curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null
-    curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null
-
-    mcount "steps.vendor.geo_libs"
-    # Copy libjasper from build image to slug.
-    if [[ "$STACK" == "heroku-16" ]]; then
-      cp /usr/lib/x86_64-linux-gnu/libjasper.so* ".heroku/vendor/lib/."
-      mcount "steps.vendor.libjasper"
-    fi
-
-  fi
-
-  GDAL=$(pwd)/vendor
-  export GDAL
-  # set path for post_compile hooks
-  export GDAL_LIBRARY_PATH="$BUILD_DIR/.heroku/vendor/lib/libgdal.so"
-  export GEOS_LIBRARY_PATH="$BUILD_DIR/.heroku/vendor/lib/libgeos_c.so"
-  # set path for runtime environmeht
-  set_env GDAL_LIBRARY_PATH "/app/.heroku/vendor/lib/libgdal.so"
-  set_env GEOS_LIBRARY_PATH "/app/.heroku/vendor/lib/libgeos_c.so"
-fi

bin/steps/mercurial

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-# Install Mercurial if it appears to be required.
-if [[ -f "requirements.txt" ]]; then
-	if (grep -Fiq "hg+" requirements.txt) then
-		/app/.heroku/python/bin/pip install  mercurial | cleanup | indent
-		mcount "steps.mercurial"
-	fi
-fi

bin/steps/pip-uninstall

@@ -16,7 +16,7 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
       mcount "failure.bad-requirements"
     fi
 
-    rm -fr requirements-declared.txt
+    rm -rf requirements-declared.txt
 
     if [[ -s .heroku/python/requirements-stale.txt ]]; then
       puts-step "Uninstalling stale dependencies"

bin/steps/pipenv-python-version

@@ -30,6 +30,9 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
                 if [ "$PYTHON" = 3.8 ]; then
                     echo "$LATEST_38" > "$BUILD_DIR/runtime.txt"
                 fi
+                if [ "$PYTHON" = 3.9 ]; then
+                    echo "$LATEST_39" > "$BUILD_DIR/runtime.txt"
+                fi
             fi
 
 

bin/steps/pylibmc

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-# This script serves as the Pylibmc build step of the
-# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
-# compiler.
-#
-# A [buildpack](https://devcenter.heroku.com/articles/buildpacks) is an
-# adapter between a Python application and Heroku's runtime.
-#
-# This script is invoked by [`bin/compile`](/).
-
-if [[ "$STACK" != "cedar-14" ]]; then
-    # libmemcached is pre-installed in the stack image so there is no need to vendor it.
-    return 0
-fi
-
-# The location of the pre-compiled libmemcached binary.
-VENDORED_MEMCACHED="${VENDOR_URL}/libraries/vendor/libmemcache.tar.gz"
-
-# Syntax sugar.
-# shellcheck source=bin/utils
-source "$BIN_DIR/utils"
-
-# If pylibmc exists within requirements, use vendored libmemcached.
-if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
-
-  if [ ! -d ".heroku/vendor/lib/sasl2" ]; then
-    echo "-----> Noticed pylibmc. Bootstrapping libmemcached."
-    mkdir -p .heroku/vendor
-    # Download and extract libmemcached into target vendor directory.
-    curl "$VENDORED_MEMCACHED" -s | tar zxv -C .heroku/vendor &> /dev/null
-    mcount "steps.vendor.pylibmc"
-  fi
-
-  LIBMEMCACHED=$(pwd)/vendor
-  export LIBMEMCACHED
-fi

bin/steps/python

@@ -5,7 +5,7 @@ runtime-fixer runtime.txt
 PYTHON_VERSION=$(cat runtime.txt)
 
 # The location of the pre-compiled python binary.
-VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"
+VENDORED_PYTHON="${S3_BASE_URL}/${STACK}/runtimes/${PYTHON_VERSION}.tar.gz"
 
 SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
 SECURITY_UPDATE_PYPY="The PyPy project has released a security update! Please consider upgrading to"
@@ -16,6 +16,12 @@ PYTHON_2_EOL_UPDATE="Python 2 has reached it's community EOL. Upgrade your Pytho
 
 # check if runtime exists
 if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
+  if [[ "$PYTHON_VERSION" == $PY39* ]]; then
+    if [ "$PYTHON_VERSION" != "$LATEST_39" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_39"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
   if [[ "$PYTHON_VERSION" == $PY38* ]]; then
     # do things to alert the user of security release available
     if [ "$PYTHON_VERSION" != "$LATEST_38" ]; then
@@ -84,13 +90,13 @@ mcount "version.python.${PYTHON_VERSION}"
 
 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
     puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache"
-    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
+    rm -rf .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
 fi
 
 if [ -f .heroku/python-version ]; then
   if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then
       puts-step "Found $(cat .heroku/python-version), removing"
-      rm -fr .heroku/python
+      rm -rf .heroku/python
   else
     SKIP_INSTALL=1
   fi
@@ -160,7 +166,7 @@ fi
 # Instead, we use the pip wheel to install itself, using the method described here:
 # https://github.com/pypa/pip/issues/2351#issuecomment-69994524
 PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl"
-PIP_WHEEL_URL="https://lang-python.s3.amazonaws.com/common/${PIP_WHEEL_FILENAME}"
+PIP_WHEEL_URL="${S3_BASE_URL}/common/${PIP_WHEEL_FILENAME}"
 PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}"
 
 if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then

bin/utils

@@ -2,12 +2,13 @@
 shopt -s extglob
 shopt -s nullglob
 
-# The standard library.
-if [[ ! -f  /tmp/stdlib.sh ]]; then
-  curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v8/stdlib.sh > /tmp/stdlib.sh
-fi
-# shellcheck source=/dev/null
-source /tmp/stdlib.sh
+# This is necessary since this script is sometimes sourced from
+# subshells that don't have the variables from bin/compile.
+# Remove this once we no longer wrap all the things in `sub_env`.
+BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT_DIR=$(dirname "${BIN_DIR}")
+# shellcheck source=vendor/buildpack-stdlib_v8.sh
+source "${ROOT_DIR}/vendor/buildpack-stdlib_v8.sh"
 
 if [ "$(uname)" == Darwin ]; then
     sed() { command sed -l "$@"; }

bin/warnings

@@ -12,16 +12,6 @@ old-platform() {
   fi
 }
 
-pylibmc-missing() {
-  if grep -qi 'fatal error: libmemcached/memcached.h: No such file or directory' "$WARNINGS_LOG"; then
-    echo
-    puts-warn "Hello! There was a problem with your build related to libmemcache."
-    puts-warn "The Python library 'pylibmc' must be explicitly specified in 'requirements.txt' in order to build correctly."
-    puts-warn "Once you do that, everything should work as expected.   -- Much Love, Heroku."
-    mcount 'warnings.libmemcache'
-  fi
-}
-
 scipy-included() {
   if grep -qi 'running setup.py install for scipy' "$WARNINGS_LOG"; then
     echo
@@ -56,11 +46,22 @@ six-included() {
   fi
 }
 
+gdal-missing() {
+  if grep -qi 'Could not find gdal-config' "$WARNINGS_LOG"; then
+    mcount 'warnings.gdal'
+    echo
+    puts-warn "Hello! Package installation failed since the GDAL library was not found."
+    puts-warn "For GDAL, GEOS and PROJ support, use the Geo buildpack alongside the Python buildpack:"
+    puts-warn "https://github.com/heroku/heroku-geo-buildpack"
+    puts-warn "  -- Much Love, Heroku."
+  fi
+}
+
 show-warnings() {
   old-platform
-  pylibmc-missing
   scipy-included
   distribute-included
   six-included
+  gdal-missing
 }
 

builds/README.md

@@ -1,57 +1,80 @@
 # Python Buildpack Binaries
 
-## Building the Docker Images
-
-**After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack:
-
-    $ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
-    $ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-    $ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-
-## Using the Image
-
-You can e.g. `bash` into each of the images you built using their tag:
-
-    docker run --rm -ti heroku-python-build-cedar-14 bash
-    docker run --rm -ti heroku-python-build-heroku-16 bash
-    docker run --rm -ti heroku-python-build-heroku-18 bash
-
-You then have a shell where you can run `bob build`, `bob deploy`, and so forth. You can of course also invoke these programs directly with `docker run`:
-
-    docker run --rm -ti heroku-python-build-heroku-18 bob build runtimes/python-2.7.15
-
-In order to `bob deploy`, AWS credentials must be set up, as well as name and prefix of your custom S3 bucket (unless you're deploying to the Heroku production buckets that are pre-defined in each `Dockerfile`); see next section for details.
+The binaries for this buildpack are built in Docker containers based on the Heroku stack image.
 
 ## Configuration
 
-File `dockerenv.default` contains a list of required env vars; most of these have default values defined in `Dockerfile`. You can copy this file to a location outside the buildpack and modify it with the values you desire and pass its location with `--env-file`, or pass the env vars to `docker run` using `--env`.
+In order to publish binaries AWS credentials must be passed to the build container.
+If you are testing only the build (ie: `bob build`), these are optional.
 
-Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCKET`, `S3_PREFIX`, and `S3_REGION`. If you're building your own packages, you'll likely want to change `S3_BUCKET` and `S3_PREFIX` to match your info. Instead of setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` into that file, you may also pass them to `docker run` through the environment, or explicitly using `--env`, in order to prevent accidental commits of credentials.
+In addition, unless you are building the official binaries for Heroku (which use the defaults
+specified in each `Dockerfile`), you will need to override `S3_BUCKET` and `S3_PREFIX` to
+match your own S3 bucket/use case.
 
-### Passing AWS credentials to the container
+If you only need to set AWS credentials, you can do so by setting the environment variables
+`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` before calling the make commands.
 
-If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file.
+For example:
 
-#### Passing credentials explicitly
+```bash
+set +o history # Disable bash history
+export AWS_ACCESS_KEY_ID=...
+export AWS_SECRET_ACCESS_KEY=...
+set -o history # Re-enable bash history
+make ...
+```
 
-    docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash
+If you need to override the default S3 bucket, or would prefer not to use credentials via
+environment variables, then you need to instead use a Docker env file like so:
 
-#### Passing credentials through  the environment
+1. Copy the `builds/dockerenv.default` env file to a location outside the buildpack repository.
+2. Edit the new file, adding at a minimum the values for the variables
+   `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (see Docker
+   [env-file documentation](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file)).
+3. Pass the path of the file to the make commands using `ENV_FILE`. For example:
 
-The two environment variables `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY` are defined in `builds/dockerenv.default`, without values. This will cause Docker to "forward" values for these variables from the current environment, so you can pass them in:
+   ```bash
+   make ... ENV_FILE=~/.dockerenv.python-buildpack
+   ```
 
-    AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+## Launching an interactive build environment
 
-or
+To start an interactive version of the build environment (ideal for development) use the
+`buildenv` make target, passing in the desired `STACK` name. For example:
 
-    export AWS_ACCESS_KEY_ID=...
-    export AWS_SECRET_ACCESS_KEY=...
-    docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+```bash
+make buildenv STACK=heroku-18
+```
 
-#### Passing credentials through a separate env file
+This will create the builder docker image based on the latest image for that stack, and
+then start a bash shell where you can run `bob build`, `bob deploy`, and so forth.
 
-This method is the easiest for users who want to build packages in their own S3 bucket, as they will have to adjust the `S3_BUCKET` and `S3_PREFIX` environment variable values anyway from their default values.
+The `builds/` directory is bind-mounted into the running container, so local build formula
+changes will appear there immediately without the need to rebuild the image.
 
-For this method, it is important to keep the credentials file in a location outside the buildpack, so that your credentials aren't accidentally committed. Copy `builds/dockerenv.default` **to a safe location outside the buildpack directory**, and insert your values for `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY`.
+## Bulk deploying runtimes
 
-    docker run --rm -ti --env-file=../SOMEPATHOUTSIDE/s3.env heroku-python-build-heroku-18 bash
+When a new Python version is released, binaries have to be generated for multiple stacks.
+To automate this, use the `deploy-runtimes` make target, which will ensure the builder
+image is up to date, and then run `bob deploy` for each runtime-stack combination.
+
+The build formula name(s) are passed using `RUNTIMES`, like so:
+
+```bash
+make deploy-runtimes RUNTIMES='python-X.Y.Z'
+```
+
+By default this will deploy to all supported stacks (see `STACKS` in `Makefile`),
+but this can be overridden using `STACKS`:
+
+```bash
+make deploy-runtimes RUNTIMES='python-X.Y.Z' STACKS='heroku-16 heroku-18'
+```
+
+Multiple runtimes can also be specified (useful for when adding a new stack), like so:
+
+```bash
+make deploy-runtimes RUNTIMES='python-A.B.C python-X.Y.Z' STACKS='heroku-20'
+```
+
+Note: Both `RUNTIMES` and `STACKS` are space delimited.

builds/dockerenv.default

@@ -1,5 +1,8 @@
+# Since no values are specified here, these variables will be read from the environment at run time:
+# https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file
 AWS_ACCESS_KEY_ID
 AWS_SECRET_ACCESS_KEY
-S3_BUCKET
-S3_PREFIX
-S3_REGION
+
+# Uncomment these if you need to override the default S3 bucket and/or path prefixes.
+# S3_BUCKET
+# S3_PREFIX

builds/heroku-16.Dockerfile

@@ -1,7 +1,7 @@
 FROM heroku/heroku:16-build
 
 ENV WORKSPACE_DIR="/app/builds" \
-    S3_BUCKET="lang-python" \
+    S3_BUCKET="heroku-buildpack-python" \
     S3_PREFIX="heroku-16/" \
     STACK="heroku-16"
 

builds/heroku-18.Dockerfile

@@ -1,7 +1,7 @@
 FROM heroku/heroku:18-build
 
 ENV WORKSPACE_DIR="/app/builds" \
-    S3_BUCKET="lang-python" \
+    S3_BUCKET="heroku-buildpack-python" \
     S3_PREFIX="heroku-18/" \
     STACK="heroku-18"
 

builds/heroku-20.Dockerfile

@@ -1,21 +1,20 @@
-FROM heroku/cedar:14
+FROM heroku/heroku:20-build
 
 ENV WORKSPACE_DIR="/app/builds" \
-    S3_BUCKET="lang-python" \
-    S3_PREFIX="cedar-14/" \
-    STACK="cedar-14"
+    S3_BUCKET="heroku-buildpack-python" \
+    S3_PREFIX="heroku-20/" \
+    STACK="heroku-20"
 
 RUN apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
         libsqlite3-dev \
         python3-pip \
-        realpath \
+        python3-setuptools \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
 COPY requirements.txt /app/
-# Can't use `--disable-pip-version-check --no-cache-dir` since not supported by Ubuntu 14.04's pip.
-RUN pip3 install -r /app/requirements.txt
+RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
 
 COPY . /app

builds/libraries/sqlite

@@ -19,4 +19,4 @@ make install
 
 # Cleanup
 cd ..
-rm -fr sqlite
+rm -rf sqlite

builds/libraries/vendor/gdal

@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building gdal…"
-
-VERSION="2.2.1"
-SOURCE_TARBALL="http://download.osgeo.org/gdal/${VERSION}/gdal-${VERSION}.tar.gz"
-
-curl -L $SOURCE_TARBALL | tar zx
-
-pushd "gdal-${VERSION}"
-./configure --prefix=$OUT_PREFIX --enable-static=no  &&
-make
-make install
-
-# Cleanup
-popd

builds/libraries/vendor/geos

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building geos…"
-
-VERSION=3.6.2
-
-SOURCE_TARBALL="http://download.osgeo.org/geos/geos-${VERSION}.tar.bz2"
-
-curl -L $SOURCE_TARBALL | tar xj
-
-pushd "geos-${VERSION}"
-./configure --prefix=$OUT_PREFIX --enable-static=no &&
-make
-make install
-
-# Cleanup
-popd

builds/libraries/vendor/libmemcache

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-if [[ $S3_PREFIX != "cedar-14" ]]; then
-    echo "libmemcached only needs to be built for cedar-14, since newer stacks include it in the base image"
-    exit 1
-fi
-
-# fail hard
-set -o pipefail
-# fail harder
-set -eux
-
-DEFAULT_VERSION="1.0.18"
-dep_version=${VERSION:-$DEFAULT_VERSION}
-dep_dirname=libmemcached-${dep_version}
-dep_archive_name=${dep_dirname}.tar.gz
-dep_url=https://launchpad.net/libmemcached/1.0/${dep_version}/+download/${dep_archive_name}
-
-# SASL Support.
-echo "-----> Building cyrus-sasl 2.1.26…"
-
-curl -LO ftp://ftp.cyrusimap.org/cyrus-sasl/cyrus-sasl-2.1.26.tar.gz
-# FTP doesn't play well with piping into tar xz
-tar xzf cyrus-sasl-2.1.26.tar.gz
-
-pushd cyrus-sasl-2.1.26
-./configure --prefix=${OUT_PREFIX} --with-plugindir=${OUT_PREFIX}lib/sasl2 --with-configdir=${OUT_PREFIX}lib/sasl2
-
-make -s -j 9
-make install -s
-popd
-
-echo "-----> Building libmemcached ${dep_version}…"
-
-curl -L ${dep_url} | tar xz
-pushd ${dep_dirname}
-CPPFLAGS=-I${OUT_PREFIX}/include LDFLAGS=-L${OUT_PREFIX}/lib ./configure --prefix=${OUT_PREFIX} --without-memcached
-make -s -j 9
-make install -s
-popd
-
-echo "-----> Done."

builds/libraries/vendor/proj

@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building gdal…"
-
-VERSION=4.9.3
-SOURCE_TARBALL="http://download.osgeo.org/proj/proj-${VERSION}.tar.gz"
-
-curl -L $SOURCE_TARBALL | tar zx
-
-pushd "proj-${VERSION}"
-./configure --prefix=$OUT_PREFIX --enable-static=no
-make
-make install
-
-# Cleanup
-popd

builds/runtimes/python-3.9.0

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

requirements.txt

@@ -1,3 +1,6 @@
-docopt==0.6.2
-bob-builder==0.0.18
+# Dependencies for generating/publishing Python binaries.
+bob-builder==0.0.19
+
+# Sub-dependencies of bob-builder.
 boto==2.49.0
+docopt==0.6.2

test/fixtures/gdal/requirements.txt

@@ -0,0 +1 @@
+gdal

test/fixtures/geos/requirements.txt

@@ -1 +0,0 @@
-django

test/fixtures/nltk/runtime.txt

@@ -1 +0,0 @@
-python-3.6.6

test/fixtures/pipenv-full-version/Pipfile

@@ -6,4 +6,4 @@ verify_ssl = true
 requests = "*"
 
 [requires]
-python_full_version = "3.6.3"
+python_full_version = "3.7.8"

test/fixtures/pipenv-full-version/Pipfile.lock

@@ -1,24 +1,11 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "22a052f4d1cfe6518b2f236fe45c3208c587a9ab1323bdd390632e27278b541e"
-        },
-        "host-environment-markers": {
-            "implementation_name": "cpython",
-            "implementation_version": "3.6.3",
-            "os_name": "posix",
-            "platform_machine": "x86_64",
-            "platform_python_implementation": "CPython",
-            "platform_release": "16.7.0",
-            "platform_system": "Darwin",
-            "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64",
-            "python_full_version": "3.6.3",
-            "python_version": "3.6",
-            "sys_platform": "darwin"
+            "sha256": "8a36860f0f9cb55716222098062cea5c5e0f8127cafb9d0c694de327bac9fbc0"
         },
         "pipfile-spec": 6,
         "requires": {
-            "python_full_version": "3.6.3"
+            "python_full_version": "3.7.8"
         },
         "sources": [
             {

test/fixtures/pypy2_7_warn/runtime.txt

@@ -1 +1 @@
-pypy2.7-7.2.0
+pypy2.7-7.3.1

test/fixtures/pypy3_6_warn/runtime.txt

@@ -1 +1 @@
-pypy3.6-7.2.0
+pypy3.6-7.3.1

test/fixtures/python2_warn/requirements.txt

@@ -1 +0,0 @@
-requests

test/fixtures/python3_4_warn/requirements.txt

@@ -1 +0,0 @@
-flask

test/fixtures/python3_5_warn/requirements.txt

@@ -1 +0,0 @@
-flask

test/fixtures/python3_6/runtime.txt

@@ -1 +1 @@
-python-3.6.8
+python-3.6.12

test/fixtures/python3_6_warn/requirements.txt

@@ -1 +0,0 @@
-requests

test/fixtures/python3_6_warn/runtime.txt

@@ -1 +1 @@
-python-3.6.7
+python-3.6.11

test/fixtures/python3_7/runtime.txt

@@ -1 +1 @@
-python-3.7.2
+python-3.7.9

test/fixtures/python3_7_warn/requirements.txt

@@ -1 +0,0 @@
-requests

test/fixtures/python3_7_warn/runtime.txt

@@ -1 +1 @@
-python-3.7.1
+python-3.7.8

test/fixtures/python3_8/runtime.txt

@@ -1 +1 @@
-python-3.8.2
+python-3.8.6

test/fixtures/python3_8_warn/requirements.txt

@@ -1 +0,0 @@
-requests

test/fixtures/python3_8_warn/runtime.txt

@@ -1 +1 @@
-python-3.8.0
+python-3.8.5

test/fixtures/python3_9/runtime.txt

@@ -0,0 +1 @@
+python-3.9.0

test/fixtures/python3_9_warn/runtime.txt

@@ -0,0 +1 @@
+python-3.9.0

test/fixtures/pythonDefault/requirements.txt

@@ -1 +0,0 @@
-flask

test/fixtures/requirements-mercurial/requirements.txt

@@ -0,0 +1 @@
+hg+https://www.mercurial-scm.org/repo/python-hglib/#egg=python-hglib

test/run-deps

@@ -17,21 +17,18 @@ testCollectstatic() {
   assertCaptured "collectstatic"
 }
 
-testGEOS() {
+testBuildWithGeoLibrariesWarning() {
   local env_dir="$(mktmpdir)"
   echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
-  compile 'geos' '' "${env_dir}"
-  assertCaptured "geos"
-  assertCapturedSuccess
+  compile 'gdal' '' "${env_dir}"
+  assertCaptured " !     The Python buildpack's BUILD_WITH_GEO_LIBRARIES functonality is no longer supported"
+  assertCapturedError
 }
 
-testGEOSDeprecation() {
-  local env_dir="$(mktmpdir)"
-  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
-  compile 'geos' '' "${env_dir}"
-  assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated.
- !     An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
-  assertCapturedSuccess
+testGDALWarning() {
+  compile 'gdal'
+  assertCaptured " !     Hello! Package installation failed since the GDAL library was not found."
+  assertCapturedError
 }
 
 testNLTK() {
@@ -43,7 +40,9 @@ testNLTK() {
   echo 'ignore::RuntimeWarning' > "${env_dir}/PYTHONWARNINGS"
   compile 'nltk' '' "${env_dir}"
   assertCaptured "[nltk_data] Downloading package city_database" "STD_ERR"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since the NLTK downloader outputs all
+  # progress/status messages to stderr (W-8146040).
+  assertCapturedSuccessWithStdErr
 }
 
 testPsycopg2() {
@@ -53,17 +52,16 @@ testPsycopg2() {
 }
 
 testPysqlite() {
+  # pysqlite does not support Python 3 (since the sqlite3 stdlib can be used there),
+  # so we have to test with Python 2, which we've not made available for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "pysqlite"
   assertCaptured "pysqlite"
   assertCapturedSuccess
 }
 
-testSqliteInstall() {
-  compile "pythonDefault"
-  assertNotCaptured "Sqlite3 failed to install."
-  assertCapturedSuccess
-}
-
 testCffi() {
   compile "cffi"
   assertCaptured "cffi"
@@ -76,6 +74,12 @@ testPylibmc() {
   assertCapturedSuccess
 }
 
+testMercurial() {
+  compile "requirements-mercurial"
+  assertCaptured "Cloning hg"
+  assertCapturedSuccess
+}
+
 pushd $(dirname 0) >/dev/null
 popd >/dev/null
 

test/run-features

@@ -33,7 +33,9 @@ testStackChange() {
 testSetupPy() {
   compile "setup-py"
   assertCaptured "maya"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
 
 testStandardRequirements() {
@@ -45,29 +47,44 @@ testStandardRequirements() {
 testPipenv() {
   compile "pipenv"
   assertCaptured "Installing pip 9.0.2, setuptools 47.1.1 and wheel 0.34.2"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
 
 testPipenvLock() {
   compile "pipenv-lock"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
 
-testPipenvVersion() {
+testPipenvPythonVersion3_6() {
   compile "pipenv-version"
-  assertCaptured $DEFAULT_PYTHON_VERSION
-  assertCapturedSuccess
+  assertCaptured "Installing ${LATEST_36}"
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
 
-testPipenvVersion2() {
+testPipenvPythonVersion2_7() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "pipenv-version2"
-  assertCaptured $LATEST_27
-  assertCapturedSuccess
+  assertCaptured "Installing ${LATEST_27}"
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
-testPipenvFullVersion() {
+
+testPipenvPythonFullVersion() {
   compile "pipenv-full-version"
-  assertCaptured "3.6.3"
-  assertCapturedSuccess
+  assertCaptured "3.7.8"
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
 
 testNoRequirements() {
@@ -95,6 +112,7 @@ testHooks() {
   local expected_env_vars=(
     _
     BIN_DIR
+    BPLOG_PREFIX
     BUILD_DIR
     BUILDPACK_LOG_FILE
     CACHE_DIR
@@ -116,9 +134,8 @@ testHooks() {
     SHLVL
     SOME_APP_CONFIG_VAR
     STACK
-    VENDOR_URL
   )
-  if [[ "${STACK}" == "cedar-14" || "${STACK}" == "heroku-16" ]]; then
+  if [[ "${STACK}" == "heroku-16" ]]; then
     # Remove "OLDPWD" from expected_env_vars since for bash <4.4 it's not exported to subshells:
     # https://github.com/heroku/heroku-buildpack-python/pull/1011#issuecomment-665117835
     read -ra expected_env_vars <<< "${expected_env_vars[@]/OLDPWD/}"

test/run-versions

@@ -4,55 +4,78 @@
 # shellcheck source=bin/default_pythons
 source "bin/default_pythons"
 
-testPythonDefault() {
-  updateVersion "pythonDefault" $DEFAULT_PYTHON_VERSION
-  compile "pythonDefault"
-  assertCaptured $DEFAULT_PYTHON_VERSION
+testPythonVersionUnspecified() {
+  compile "python_version_unspecified"
+  assertCaptured "Installing ${DEFAULT_PYTHON_VERSION}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
 
-testPython2() {
-    updateVersion "python2" $LATEST_27
-    echo $LATEST_27 > "runtime.txt"
-    compile "python2"
-    assertCaptured $LATEST_27
-    assertCaptured "python-2-7-eol-faq";
-    assertNotCaptured "security update"
-    assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+testPython2_7() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
+  compile "python2"
+  assertCaptured "Installing ${LATEST_27}"
+  assertCaptured "python-2-7-eol-faq";
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }
 
-testPython2_warn() {
-    compile "python2_warn"
-    assertCaptured "python-2.7.15"
-    assertCaptured "python-2-7-eol-faq";
-    assertCaptured "Only the latest version"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+testPython2_7_warn() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
+  compile "python2_warn"
+  assertCaptured "Installing python-2.7.15"
+  assertCaptured "python-2-7-eol-faq";
+  assertCaptured "Only the latest version"
+  assertCaptured "${LATEST_27}"
+  assertCapturedSuccess
 }
 
 testPython3_4() {
+  # Python 3.4 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_4"
-  assertCaptured $LATEST_34
+  assertCaptured "Installing ${LATEST_34}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 19.1.1, setuptools 43.0.0 and wheel 0.33.6"
-  assertCapturedSuccess
+  assertCaptured "Installing SQLite3"
+  # Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
+  # and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
+  assertCapturedSuccessWithStdErr
 }
 
 testPython3_4_warn() {
+  # Python 3.4 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_4_warn"
-  assertCaptured "python-3.4.9"
+  assertCaptured "Installing python-3.4.9"
   assertCaptured "security update!"
-  assertCapturedSuccess
+  assertCaptured "${LATEST_34}"
+  # Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
+  # and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
+  assertCapturedSuccessWithStdErr
 }
 
 testPython3_5() {
+  # Python 3.5 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_5"
-  assertCaptured $LATEST_35
+  assertCaptured "Installing ${LATEST_35}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
@@ -60,16 +83,20 @@ testPython3_5() {
 }
 
 testPython3_5_warn() {
+  # Python 3.5 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
   compile "python3_5_warn"
-  assertCaptured "python-3.5.6"
+  assertCaptured "Installing python-3.5.6"
   assertCaptured "security update!"
+  assertCaptured "${LATEST_35}"
   assertCapturedSuccess
 }
 
 testPython3_6() {
-  updateVersion "python3_6" $LATEST_36
   compile "python3_6"
-  assertCaptured $LATEST_36
+  assertCaptured "Installing ${LATEST_36}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
@@ -78,62 +105,65 @@ testPython3_6() {
 
 testPython3_6_warn() {
   compile "python3_6_warn"
-  assertCaptured "python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
   assertCaptured "security update!"
-  assertCaptured "Installing SQLite3"
+  assertCaptured "${LATEST_36}"
   assertCapturedSuccess
 }
 
 testPython3_7() {
-  updateVersion "python3_7" $LATEST_37
   compile "python3_7"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertNotCaptured "security update"
-    assertCaptured $LATEST_37
-    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing ${LATEST_37}"
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }
 
 testPython3_7_warn() {
   compile "python3_7_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "python-3.7.1"
-    assertCaptured "security update!"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing python-3.7.8"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_37}"
+  assertCapturedSuccess
 }
 
 testPython3_8() {
-  updateVersion "python3_8" $LATEST_38
   compile "python3_8"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertNotCaptured "security update"
-    assertCaptured $LATEST_38
-    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing ${LATEST_38}"
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }
 
 testPython3_8_warn() {
   compile "python3_8_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "python-3.8.0"
-    assertCaptured "security update!"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+  assertCaptured "Installing python-3.8.5"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_38}"
+  assertCapturedSuccess
+}
+
+testPython3_9() {
+  compile "python3_9"
+  assertCaptured "Installing ${LATEST_39}"
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
+}
+
+testPython3_9_warn() {
+  # Can't test the version warning until there is at least one old version of Python 3.9.
+  if [[ "${LATEST_39}" = "python-3.9.0" ]]; then
+    return
   fi
+  compile "python3_9_warn"
+  assertCaptured "Installing python-3.9.0"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_39}"
+  assertCapturedSuccess
 }
 
 testPythonVersionInvalid() {
@@ -145,67 +175,58 @@ testPythonVersionInvalid() {
 
 testPypy3_6() {
   compile "pypy3_6"
-  assertCaptured "Installing pypy"
+  assertCaptured "Installing ${LATEST_PYPY_36}"
   assertNotCaptured "security update"
-  assertCaptured "$LATEST_PYPY_36"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCapturedSuccess
 }
 
 testPypy3_6_warn() {
   compile "pypy3_6_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "Installing pypy"
-    assertCaptured "security update!"
-    assertCaptured "$LATEST_PYPY_36"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing pypy3.6-7.3.1"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_PYPY_36}"
+  assertCapturedSuccess
 }
 
 testPypy2_7() {
   compile "pypy2_7"
-  assertCaptured "Installing pypy"
+  assertCaptured "Installing ${LATEST_PYPY_27}"
   assertNotCaptured "security update"
-  assertCaptured "$LATEST_PYPY_27"
   assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
   assertCapturedSuccess
 }
 
 testPypy2_7_warn() {
   compile "pypy2_7_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "Installing pypy"
-    assertCaptured "security update!"
-    assertCaptured "$LATEST_PYPY_27"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing pypy2.7-7.3.1"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_PYPY_27}"
+  assertCapturedSuccess
 }
 
 testStickyPythonVersion() {
   local cache_dir="$(mktmpdir)"
   compile "python3_6_warn" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
   assertCapturedSuccess
-  compile "no-runtime-txt" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  compile "python_version_unspecified" "$cache_dir"
+  assertNotCaptured "Installing python"
+  assertCaptured "security update!"
   assertCapturedSuccess
   # Whilst this file seems like an implementation detail (so something that should
   # not be tested), we must guarantee the filename remains consistent for backwards
   # compatibility across buildpack versions for already-built apps.
-  assertFile "python-3.6.7" ".heroku/python-version"
+  assertFile "python-3.6.11" ".heroku/python-version"
 }
 
 testPythonVersionChange() {
   local cache_dir="$(mktmpdir)"
   compile "python3_6_warn" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
   assertCapturedSuccess
   compile "python3_6" "$cache_dir"
-  assertCaptured "Found python-3.6.7, removing"
+  assertCaptured "Found python-3.6.11, removing"
   assertCapturedSuccess
 }
 

test/utils

@@ -56,11 +56,6 @@ resetCapture()
   unset rtrn # deprecated
 }
 
-updateVersion()
-{
-  echo "$2" > "test/fixtures/${1}/runtime.txt"
-}
-
 assertCapturedEquals()
 {
   assertEquals "$@" "$(cat ${STD_OUT})"
@@ -85,12 +80,20 @@ assertNotCaptured()
 assertCapturedSuccess()
 {
   assertEquals "Captured exit code -" "0" "${RETURN}"
-  # assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
+  assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
 
-  if [ $RETURN -ne 0 -a -z "$(cat ${STD_ERR})" ]; then
-    # Failing exit code but stderr was empty. Display stdout to help debugging.
-    cat $STD_OUT
-    echo
+  if [ $RETURN -ne 0 -o -n "$(cat ${STD_ERR})" ]; then
+    debug
+  fi
+}
+
+assertCapturedSuccessWithStdErr()
+{
+  assertEquals "Captured exit code -" "0" "${RETURN}"
+  assertNotEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
+
+  if [ ${RETURN} -ne 0 ]; then
+    debug
   fi
 }
 
@@ -151,9 +154,14 @@ _assertContains()
 
 debug()
 {
-  cat $STD_OUT
-  echo '^^^^^^'
-  cat $STD_ERR
+  echo
+  echo '### STD_OUT ###'
+  cat "${STD_OUT}"
+  echo
+  echo '### STD_ERR ###'
+  cat "${STD_ERR}"
+  echo
+  echo
 }
 
 assertContains()

vendor/buildpack-stdlib_v8.sh

@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+
+# From:
+# https://raw.githubusercontent.com/heroku/buildpack-stdlib/v8/stdlib.sh
+
+# Buildpack defaults
+# ---------------
+
+export BUILDPACK_LOG_FILE="${BUILDPACK_LOG_FILE:-/dev/null}"
+
+# Standard Output
+# ---------------
+
+# Buildpack Steps.
+puts_step() {
+  if [[ "$*" == "-" ]]; then
+    read -r output
+  else
+    output=$*
+  fi
+  echo -e "\\e[1m\\e[36m=== $output\\e[0m"
+  unset output
+}
+
+# Buildpack Error.
+puts_error() {
+  if [[ "$*" == "-" ]]; then
+    read -r output
+  else
+    output=$*
+  fi
+  echo -e "\\e[1m\\e[31m=!= $output\\e[0m"
+}
+
+# Buildpack Warning.
+puts_warn() {
+  if [[ "$*" == "-" ]]; then
+    read -r output
+  else
+    output=$*
+  fi
+  echo -e "\\e[1m\\e[33m=!= $output\\e[0m"
+}
+
+# Is verbose set?
+is_verbose() {
+  if [[ -n $BUILDPACK_VERBOSE ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+# Buildpack Verbose.
+puts_verbose() {
+  if is_verbose; then
+    if [[ "$*" == "-" ]]; then
+      read -r output
+    else
+      output=$*
+    fi
+    echo "$output"
+    unset output
+  fi
+}
+
+# Buildpack Utilities
+# -------------------
+
+# Usage: $ set-env key value
+# NOTICE: Expects PROFILE_PATH & EXPORT_PATH to be set!
+set_env() {
+  # TODO: automatically create profile path directory if it doesn't exist.
+  echo "export $1=$2" >> "$PROFILE_PATH"
+  echo "export $1=$2" >> "$EXPORT_PATH"
+}
+
+# Usage: $ set-default-env key value
+# NOTICE: Expects PROFILE_PATH & EXPORT_PATH to be set!
+set_default_env() {
+  echo "export $1=\${$1:-$2}" >> "$PROFILE_PATH"
+  echo "export $1=\${$1:-$2}" >> "$EXPORT_PATH"
+}
+
+# Usage: $ un-set-env key
+# NOTICE: Expects PROFILE_PATH to be set!
+un_set_env() {
+  echo "unset $1" >> "$PROFILE_PATH"
+}
+
+# Usage: $ _env-blacklist pattern
+# Outputs a regex of default blacklist env vars.
+_env_blacklist() {
+  local regex=${1:-''}
+  if [ -n "$regex" ]; then
+    regex="|$regex"
+  fi
+  echo "^(PATH|GIT_DIR|CPATH|CPPATH|LD_PRELOAD|LIBRARY_PATH$regex)$"
+}
+
+# Usage: $ export-env ENV_DIR WHITELIST BLACKLIST
+# Exports the environment variables defined in the given directory.
+export_env() {
+  local env_dir=${1:-$ENV_DIR}
+  local whitelist=${2:-''}
+  local blacklist
+  blacklist="$(_env_blacklist "$3")"
+  if [ -d "$env_dir" ]; then
+    # Environment variable names won't contain characters affected by:
+    # shellcheck disable=SC2045
+    for e in $(ls "$env_dir"); do
+      echo "$e" | grep -E "$whitelist" | grep -qvE "$blacklist" &&
+      export "$e=$(cat "$env_dir/$e")"
+      :
+    done
+  fi
+}
+
+# Usage: $ sub-env command
+# Runs a subshell of specified command with user-provided config.
+# NOTICE: Expects ENV_DIR to be set. WHITELIST & BLACKLIST are optional.
+# Examples:
+#    WHITELIST=${2:-''}
+#    BLACKLIST=${3:-'^(GIT_DIR|PYTHONHOME|LD_LIBRARY_PATH|LIBRARY_PATH|PATH)$'}
+sub_env() {
+  (
+    # TODO: Fix https://github.com/heroku/buildpack-stdlib/issues/37
+    # shellcheck disable=SC2153
+    export_env "$ENV_DIR" "$WHITELIST" "$BLACKLIST"
+
+    "$@"
+  )
+}
+
+# Logging
+# -------
+
+# Notice: These functions expect BPLOG_PREFIX and BUILDPACK_LOG_FILE to be defined (BUILDPACK_LOG_FILE can point to /dev/null if not provided by the buildpack).
+# Example: BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}; BPLOG_PREFIX="buildpack.go"
+
+# Returns now, in milleseconds. Useful for logging.
+# Example: $ let start=$(nowms); sleep 30; mtime "glide.install.time" "${start}"
+nowms() {
+  date +%s%3N
+}
+
+# Log arbitrary data to the logfile (e.g. a packaging file).
+# Usage: $ bplog "$(<${vendorJSON})
+bplog() {
+  echo -n "${@}" | awk 'BEGIN {printf "msg=\""; f="%s"} {gsub(/"/, "\\\"", $0); printf f, $0} {if (NR == 1) f="\\n%s" } END { print "\"" }' >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Measures time elapsed for a specific build step.
+# Usage: $ let start=$(nowms); mtime "glide.install.time" "${start}"
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#distributions-measure
+mtime() {
+  local key="${BPLOG_PREFIX}.${1}"
+  local start="${2}"
+  local end="${3:-$(nowms)}"
+  echo "${key} ${start} ${end}" | awk '{ printf "measure#%s=%.3f\n", $1, ($3 - $2)/1000 }' >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Logs a count for a specific built step.
+# Usage: $ mcount "tool.govendor"
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#counting-count
+mcount() {
+  local k="${BPLOG_PREFIX}.${1}"
+  local v="${2:-1}"
+  echo "count#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Logs a measure for a specific build step.
+# Usage: $ mmeasure "tool.installed_dependencies" 42
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#distributions-measure
+mmeasure() {
+  local k="${BPLOG_PREFIX}.${1}"
+  local v="${2}"
+  echo "measure#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Logs a unuique measurement build step.
+# Usage: $ munique "versions.count" 2.7.13
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#uniques-unique
+munique() {
+  local k="${BPLOG_PREFIX}.${1}"
+  local v="${2}"
+  echo "unique#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Measures when an exit path to the buildpack is reached, given a name, then exits 1.
+# Usage: $ mcount-exi "binExists"
+mcount_exit() {
+  mcount "error.${1}"
+  exit 1
+}