Check Changelog improvements (#1128 )

The Check Changelog GitHub action now: * is skipped for dependabot PRs * uses a more descriptive job name than `check` Closes @W-8052460@. Closes @W-8098931@. [skip changelog]
Exclude more files when publishing the buildpack (#1127 )
2026-06-05 23:10:16 +00:00 · 2020-11-30 13:13:14 +00:00 · 2020-11-24 08:55:49 +00:00 · 2020-11-18 13:26:39 +00:00 · 2020-11-17 15:19:48 +00:00 · 2020-11-13 13:10:52 +00:00
78 changed files with 736 additions and 2613 deletions
@@ -5,12 +5,13 @@ on:
    types: [opened, reopened, edited, synchronize]

 jobs:
-  check:
+  check-changelog:
    runs-on: ubuntu-latest
    if: |
      !contains(github.event.pull_request.body, '[skip changelog]') &&
      !contains(github.event.pull_request.body, '[changelog skip]') &&
-      !contains(github.event.pull_request.body, '[skip ci]')
+      !contains(github.event.pull_request.body, '[skip ci]') &&
+      !contains(github.event.pull_request.labels.*.name, 'c: dependencies')
    steps:
      - uses: actions/checkout@v1
      - name: Check that CHANGELOG is touched
@@ -23,10 +23,6 @@ jobs:

 env:
  jobs:
-    - STACK=cedar-14 TEST_CMD=test/run-deps
-    - STACK=cedar-14 TEST_CMD=test/run-versions
-    - STACK=cedar-14 TEST_CMD=test/run-features
-
    - STACK=heroku-16 TEST_CMD=test/run-deps
    - STACK=heroku-16 TEST_CMD=test/run-versions
    - STACK=heroku-16 TEST_CMD=test/run-features
@@ -34,6 +30,10 @@ env:
    - STACK=heroku-18 TEST_CMD=test/run-deps
    - STACK=heroku-18 TEST_CMD=test/run-versions
    - STACK=heroku-18 TEST_CMD=test/run-features
+
+    - STACK=heroku-20 TEST_CMD=test/run-deps
+    - STACK=heroku-20 TEST_CMD=test/run-versions
+    - STACK=heroku-20 TEST_CMD=test/run-features
  global:
    - HATCHET_RETRIES=3
    - IS_RUNNING_ON_CI=true
@@ -3,6 +3,43 @@
 ## Unreleased


+## v186 (2020-11-18)
+
+- Update the `BUILD_WITH_GEO_LIBRARIES` error message (#1121).
+- Switch NLTK feature detection away from `sp-grep` (#1119).
+- Switch Django collectstatic feature detection away from `sp-grep` (#1119).
+- Remove vendored `sp-grep` script (#1119).
+- Remove vendored `pip-diff` script (#1118).
+- Remove vendored `pip-grep` script (#1116).
+
+## v185 (2020-11-12)
+
+- Error if the unsupported `BUILD_WITH_GEO_LIBRARIES` env var is set (#1115).
+- Remove deprecated GDAL/GEOS/PROJ support (#1113).
+- Remove vendored `jq` binary (#1112).
+- Remove redundant Mercurial install step (#1111).
+- Remove support for the Cedar-14 stack (#1110).
+
+## v184 (2020-10-21)
+
+- Vendor buildpack-stdlib instead of fetching from S3 (#1100).
+- Fix metric names for metrics emitted within `sub_env` (#1099).
+
+## v183 (2020-10-12)
+
+- Add support for Heroku-20 (#968).
+
+## v182 (2020-10-06)
+
+- Python 3.9.0 is now available (CPython) (#1090).
+- Migrate from the `lang-python` S3 bucket to `heroku-buildpack-python` (#1089).
+- Remove `vendor/shunit2` (#1086).
+- Replace `BUILDPACK_VENDOR_URL` and `USE_STAGING_BINARIES` with `BUILDPACK_S3_BASE_URL` (#1085).
+
+## v181 (2020-09-29)
+
+- PyPy 2.7 and 3.6, version 7.3.2 are now available (Note: PyPy support is in beta) (#1081).
+
 ## v180 (2020-09-24)

 - Python 3.8.6 is now available (CPython) (#1072).
@@ -3,17 +3,17 @@ GEM
  specs:
    diff-lcs (1.4.4)
    erubis (2.7.0)
-    excon (0.76.0)
+    excon (0.78.0)
    heroics (0.1.1)
      erubis (~> 2.0)
      excon
      moneta
      multi_json (>= 1.9.2)
-    heroku_hatchet (7.2.0)
+    heroku_hatchet (7.3.3)
      excon (~> 0)
      platform-api (~> 3)
      rrrretry (~> 1)
-      thor (~> 0)
+      thor (~> 1)
      threaded (~> 0)
    moneta (1.0.0)
    multi_json (1.15.0)
@@ -28,22 +28,22 @@ GEM
    rake (13.0.1)
    rate_throttle_client (0.1.2)
    rrrretry (1.0.0)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.0)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.0)
      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.0)
      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
+      rspec-support (~> 3.10.0)
    rspec-retry (0.6.2)
      rspec-core (> 3.3)
-    rspec-support (3.9.3)
-    thor (0.20.3)
+    rspec-support (3.10.0)
+    thor (1.0.1)
    threaded (0.0.4)

 PLATFORMS
@@ -1,55 +1,61 @@
 # These targets are not files
-.PHONY: check test buildenv-heroku-16 buildenv-heroku-18 tools
+.PHONY: check test compile builder-image buildenv deploy-runtimes tools

 STACK ?= heroku-18
+STACKS ?= heroku-16 heroku-18 heroku-20
 TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
+FIXTURE ?= test/fixtures/requirements-standard
+ENV_FILE ?= builds/dockerenv.default
+BUILDER_IMAGE_PREFIX := heroku-python-build

-ifeq ($(STACK),cedar-14)
-	# Cedar-14 doesn't have a build image varient.
-	IMAGE_TAG := heroku/cedar:14
-else
-	# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
-	IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
-endif
+# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
+STACK_IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build

 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
-	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
+	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/nltk bin/steps/pip-install bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/python
 	@shellcheck -x bin/steps/hooks/*

 test:
 	@echo "Running tests using: STACK=$(STACK) TEST_CMD='$(TEST_CMD)'"
-	@echo ""
-	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
-	@echo ""
-
-buildenv-heroku-16:
-	@echo "Creating build environment (heroku-16)..."
 	@echo
-	@docker build --pull -f $(shell pwd)/builds/heroku-16.Dockerfile -t python-buildenv-heroku-16 .
+	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(STACK_IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
+	@echo
+
+compile:
+	@echo "Running compile using: STACK=$(STACK) FIXTURE=$(FIXTURE)"
+	@echo
+	@docker run --rm -it -v $(PWD):/src:ro -e "STACK=$(STACK)" -w /buildpack "$(STACK_IMAGE_TAG)" \
+		bash -c 'cp -r /src/{bin,vendor} /buildpack && cp -r /src/$(FIXTURE) /build && mkdir /cache /env && bin/compile /build /cache /env'
+	@echo
+
+builder-image:
+	@echo "Generating binary builder image for $(STACK)..."
+	@echo
+	@docker build --pull -f builds/$(STACK).Dockerfile -t "$(BUILDER_IMAGE_PREFIX)-$(STACK)" .
+	@echo
+
+buildenv: builder-image
+	@echo "Starting build environment for $(STACK)..."
 	@echo
 	@echo "Usage..."
 	@echo
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ bob build runtimes/python-2.7.13"
-	@echo "  $$ bob deploy runtimes/python-2.7.13"
+	@echo "  $$ bob build runtimes/python-X.Y.Z"
 	@echo
-	@docker run -it --rm python-buildenv-heroku-16
+	@docker run --rm -it --env-file="$(ENV_FILE)" -v $(PWD)/builds:/app/builds "$(BUILDER_IMAGE_PREFIX)-$(STACK)" bash

-buildenv-heroku-18:
-	@echo "Creating build environment (heroku-18)..."
+deploy-runtimes:
+ifndef RUNTIMES
+	$(error No runtimes specified! Use: "make deploy-runtimes RUNTIMES='python-X.Y.Z ...' [STACKS='heroku-18 ...'] [ENV_FILE=...]")
+endif
+	@echo "Using: RUNTIMES='$(RUNTIMES)' STACKS='$(STACKS)' ENV_FILE='$(ENV_FILE)'"
 	@echo
-	@docker build --pull -f $(shell pwd)/builds/heroku-18.Dockerfile -t python-buildenv-heroku-18 .
-	@echo
-	@echo "Usage..."
-	@echo
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ bob build runtimes/python-2.7.13"
-	@echo "  $$ bob deploy runtimes/python-2.7.13"
-	@echo
-	@docker run -it --rm python-buildenv-heroku-18
-
-tools:
-	git clone https://github.com/kennethreitz/pip-pop.git
-	mv pip-pop/bin/* vendor/pip-pop/
-	rm -fr pip-pop
+	@set -eu; for stack in $(STACKS); do \
+		$(MAKE) builder-image STACK=$${stack}; \
+		for runtime in $(RUNTIMES); do \
+			echo "Generating/deploying $${runtime} for $${stack}..."; \
+			echo; \
+			docker run --rm -it --env-file="$(ENV_FILE)" "$(BUILDER_IMAGE_PREFIX)-$${stack}" bob deploy "runtimes/$${runtime}"; \
+			echo; \
+		done; \
+	done
@@ -1,27 +1,5 @@
 This buildpack includes some vendorized packages to ease installation.

-jq license
----------
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-
 shunit2 license
 ---------------

@@ -85,28 +63,3 @@ The Free Software Foundation may publish revised and/or new versions of the GNU
 Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation.

 If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library.
-
-pip-pop license
---------------
-
-The MIT License (MIT)
-
-Copyright (c) 2014 Kenneth Reitz.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
@@ -62,6 +62,7 @@ Specify a Python Runtime

 Supported runtime options include:

+- `python-3.9.0`
 - `python-3.8.6`
 - `python-3.7.9`
 - `python-3.6.12`
@@ -15,10 +15,8 @@
 # Fail fast and fail hard.
 set -eo pipefail

-# Boostrap the Buildpack Standard Library.
-# Disable unused env var warning since shellcheck doesn't know about the stdlib.
-# shellcheck disable=2034
-BPLOG_PREFIX="buildpack.python"
+# Used by buildpack-stdlib's metrics features.
+export BPLOG_PREFIX="buildpack.python"
 export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}

 [ "$BUILDPACK_XTRACE" ] && set -o xtrace
@@ -37,23 +35,19 @@ ENV_DIR=$3
 # Export Path variables, for use in sub-scripts.
 export BUILD_DIR CACHE_DIR ENV_DIR

-# Set the Buildpack's internet target for downloading Python distributions.
-# The user can provide BUILDPACK_VENDOR_URL to specify a custom target.
+# Set the base URL for downloading buildpack assets like Python runtimes.
+# The user can provide BUILDPACK_S3_BASE_URL to specify a custom target.
 # Note: this is designed for non-Heroku use, as it does not use the user-provided
 # environment variable mechanism (the ENV_DIR).
-VENDOR_URL="https://lang-python.s3.amazonaws.com/$STACK"
-if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then
-    VENDOR_URL="$BUILDPACK_VENDOR_URL"
-elif [[ -n ${USE_STAGING_BINARIES} ]]; then
-    VENDOR_URL="$USE_STAGING_BINARIES/$STACK"
-fi
-export VENDOR_URL
+S3_BASE_URL="${BUILDPACK_S3_BASE_URL:-"https://heroku-buildpack-python.s3.amazonaws.com"}"
+# This has to be exported since it's used by the geo-libs step which is run in a subshell.

 # Default Python Versions
 # shellcheck source=bin/default_pythons
 source "$BIN_DIR/default_pythons"

 # Supported Python Branches
+PY39="python-3.9"
 PY38="python-3.8"
 PY37="python-3.7"
 PY36="python-3.6"
@@ -64,7 +58,8 @@ PYPY27="pypy2.7"
 PYPY36="pypy3.6"

 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
-DEFAULT_PYTHON_STACK="cedar-14"
+# TODO: Remove this and require that STACK be set explicitly.
+DEFAULT_PYTHON_STACK="heroku-18"

 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.
@@ -73,24 +68,18 @@ DEFAULT_PYTHON_STACK="cedar-14"
 WARNINGS_LOG=$(mktemp)
 RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION

-# The buildpack ships with a few executable tools (e.g. pip-grep, etc).
+# The buildpack ships with a few executable tools.
 # This installs them into the path, so we can execute them directly.
-export PATH=$PATH:$ROOT_DIR/vendor/:$ROOT_DIR/vendor/pip-pop
+export PATH=$PATH:$ROOT_DIR/vendor/

 # Set environment variables if they weren't set by the platform.
-# Note: this is legacy, for a deprecated build system known as Anvil.
-# This can likely be removed, with caution.
-[ ! "$SLUG_ID" ] && SLUG_ID="defaultslug"
-[ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID
 [ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK

 # Sanitize externally-provided environment variables:
 # The following environment variables are either problematic or simply unneccessary
 # for the buildpack to have knowledge of, so we unset them, to keep the environment
 # as clean and pristine as possible.
-unset GIT_DIR PYTHONHOME PYTHONPATH
-unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN
-unset CYTOKINE_LOG_FILE GEM_PATH
+unset PYTHONHOME PYTHONPATH

 # Import the utils script, which contains helper functions used throughout the buildpack.
 # shellcheck source=bin/utils
@@ -101,6 +90,33 @@ source "$BIN_DIR/utils"
 # shellcheck source=bin/warnings
 source "$BIN_DIR/warnings"

+if [[ "${STACK}" == "cedar-14" ]]; then
+  mcount "failure.unsupported.cedar-14"
+  puts-warn "The Cedar-14 stack is no longer supported by the latest release of this buildpack."
+  puts-warn
+  puts-warn "Please switch to the Cedar-14 support branch by using this buildpack URL:"
+  puts-warn "https://github.com/heroku/heroku-buildpack-python#cedar-14"
+  puts-warn
+  puts-warn "For instructions on how to change the buildpacks used by an app, see:"
+  puts-warn "https://devcenter.heroku.com/articles/buildpacks#setting-a-buildpack-on-an-application"
+  exit 1
+fi
+
+if [[ -f "${ENV_DIR}/BUILD_WITH_GEO_LIBRARIES" ]]; then
+  mcount "failure.unsupported.BUILD_WITH_GEO_LIBRARIES"
+  puts-warn "The Python buildpack's legacy BUILD_WITH_GEO_LIBRARIES functonality is"
+  puts-warn "no longer supported:"
+  puts-warn "https://devcenter.heroku.com/changelog-items/1947"
+  puts-warn
+  puts-warn "To continue to use GDAL, GEOS or PROJ support, see the migration guide:"
+  puts-warn "https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-no-longer-supported"
+  puts-warn
+  puts-warn "Or if you no longer need those libraries, this message can be hidden by"
+  puts-warn "unsetting the BUILD_WITH_GEO_LIBRARIES environment variable, using:"
+  puts-warn "heroku config:unset BUILD_WITH_GEO_LIBRARIES"
+  exit 1
+fi
+
 # Make the directory in which we will create symlinks from the temporary build directory
 # to `/app`.
 # Symlinks are required, since Python is not a portable installation.
@@ -129,8 +145,9 @@ export PYTHONUNBUFFERED=1
 # Set the locale to a well-known and expected standard.
 export LANG=en_US.UTF-8
 # `~/.heroku/vendor` is an place where the buildpack may stick pre-build binaries for known
-# C dependencies (e.g. libmemcached on cedar-14). This section configures Python (GCC, more specifically)
+# C dependencies. This section configures Python (GCC, more specifically)
 # and pip to automatically include these paths when building binaries.
+# TODO: Stop adding .heroku/vendor here now that the buildpack no longer vendors anything.
 export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
 export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
 export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
@@ -170,8 +187,6 @@ cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
 cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
 # A plain text file which contains the current sqlite3 version being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-sqlite3-version" .heroku/ &> /dev/null || true
-# Any pre-compiled binaries, provided by the buildpack.
-cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true
 # "editable" installations of code repositories, via pip or pipenv.
 if [[ -d "$CACHE_DIR/.heroku/src" ]]; then
  cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true
@@ -257,27 +272,6 @@ fi
 # shellcheck source=bin/steps/eggpath-fix
 source "$BIN_DIR/steps/eggpath-fix"

-# Mercurial support.
-# If a customer appears to be using mercurial for dependency resolution, we install it first.
-# Note: this only applies to pip, not pipenv. This can likely be removed, over time. Measure it first.
-# shellcheck source=bin/steps/mercurial
-source "$BIN_DIR/steps/mercurial"
-
-# Pylibmc support.
-# On cedar-14, libmemcached was not available. The buildpack provides its own version, instead.
-# shellcheck source=bin/steps/pylibmc
-source "$BIN_DIR/steps/pylibmc"
-
-# Support for Geo libraries. This is deprecated functionality, only functional on cedar-14.
-# It is undocumented.
-# shellcheck source=bin/steps/geo-libs
-sub_env "$BIN_DIR/steps/geo-libs"
-
-# GDAL support.
-# This is part of the Geo support.
-# shellcheck source=bin/steps/gdal
-source "$BIN_DIR/steps/gdal"
-
 # SQLite3 support.
 # This sets up and installs sqlite3 dev headers and the sqlite3 binary but not the
 # libsqlite3-0 library since that exists on the stack image.
@@ -309,7 +303,7 @@ mtime "nltk.download.time" "${start}"
 # and copying it into the proper place (the logical place to do this was early, but it must be done here).
 # In CI, $BUILD_DIR is /app.
 if [[ ! "$BUILD_DIR" == "/app" ]]; then
-  rm -fr "$BUILD_DIR/.heroku/src"
+  rm -rf "$BUILD_DIR/.heroku/src"
  deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src"
 fi

@@ -375,7 +369,6 @@ mkdir -p "$CACHE_DIR/.heroku"
 cp -R .heroku/python "$CACHE_DIR/.heroku/"
 cp -R .heroku/python-version "$CACHE_DIR/.heroku/"
 cp -R .heroku/python-stack "$CACHE_DIR/.heroku/" &> /dev/null || true
-cp -R .heroku/vendor "$CACHE_DIR/.heroku/" &> /dev/null || true
 if [[ -d .heroku/src ]]; then
  cp -R .heroku/src "$CACHE_DIR/.heroku/" &> /dev/null || true
 fi
@@ -6,11 +6,12 @@
 # shellcheck disable=2034

 DEFAULT_PYTHON_VERSION="python-3.6.12"
+LATEST_39="python-3.9.0"
 LATEST_38="python-3.8.6"
 LATEST_37="python-3.7.9"
 LATEST_36="python-3.6.12"
 LATEST_35="python-3.5.10"
 LATEST_34="python-3.4.10"
 LATEST_27="python-2.7.18"
-LATEST_PYPY_36="pypy3.6-7.3.1"
-LATEST_PYPY_27="pypy2.7-7.3.1"
+LATEST_PYPY_36="pypy3.6-7.3.2"
+LATEST_PYPY_27="pypy2.7-7.3.2"
@@ -20,8 +20,8 @@ MANAGE_FILE=${MANAGE_FILE:-fakepath}
 # Legacy file-based support for $DISABLE_COLLECTSTATIC
 [ -f .heroku/collectstatic_disabled ] && DISABLE_COLLECTSTATIC=1

-# Ensure that Django is explicitly specified in requirements.txt
-sp-grep -s django && DJANGO_INSTALLED=1
+# Ensure that Django is actually installed.
+is_module_available 'django' && DJANGO_INSTALLED=1


 if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then
@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-# This script serves as the GDAL build step of the
-# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
-# compiler.
-#
-# A [buildpack](https://devcenter.heroku.com/articles/buildpacks) is an
-# adapter between a Python application and Heroku's runtime.
-#
-# This script is invoked by [`bin/compile`](/).
-
-# The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
-
-PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
-
-# Syntax sugar.
-# shellcheck source=bin/utils
-source "$BIN_DIR/utils"
-
-# If GDAL exists within requirements, use vendored gdal.
-if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
-
-  if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then
-
-    puts-warn "The vendored GDAL package in the Heroku Python Buildpack now deprecated."
-    puts-warn "To enable GDAL use an alternative buildpack is available here - https://github.com/heroku/heroku-geo-buildpack"
-
-    echo "-----> Noticed GDAL. Bootstrapping gdal."
-    mkdir -p .heroku/vendor
-    # Download and extract cryptography into target vendor directory.
-    curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
-    mcount "steps.vendor.gdal"
-  fi
-
-  GDAL=$(pwd)/vendor
-  export GDAL
-fi
@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-
-# This script serves as the GDAL build step of the
-# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
-# compiler.
-#
-# A [buildpack](https://devcenter.heroku.com/articles/buildpacks) is an
-# adapter between a Python application and Heroku's runtime.
-#
-# This script is invoked by [`bin/compile`](/).
-
-# The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
-VENDORED_GEOS="${VENDOR_URL}/libraries/vendor/geos.tar.gz"
-VENDORED_PROJ="${VENDOR_URL}/libraries/vendor/proj.tar.gz"
-
-PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
-
-# Syntax sugar.
-# shellcheck source=bin/utils
-source "$BIN_DIR/utils"
-
-# If GDAL exists within requirements, use vendored gdal.
-if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
-  mcount "buildvar.BUILD_WITH_GEO_LIBRARIES"
-
-  puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."
-  puts-warn "An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
-
-  if [ ! -f ".heroku/vendor/bin/proj" ]; then
-    echo "-----> Bootstrapping gdal, geos, proj."
-    mkdir -p .heroku/vendor
-    # Download and extract cryptography into target vendor directory.
-    curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
-    curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null
-    curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null
-
-    mcount "steps.vendor.geo_libs"
-    # Copy libjasper from build image to slug.
-    if [[ "$STACK" == "heroku-16" ]]; then
-      cp /usr/lib/x86_64-linux-gnu/libjasper.so* ".heroku/vendor/lib/."
-      mcount "steps.vendor.libjasper"
-    fi
-
-  fi
-
-  GDAL=$(pwd)/vendor
-  export GDAL
-  # set path for post_compile hooks
-  export GDAL_LIBRARY_PATH="$BUILD_DIR/.heroku/vendor/lib/libgdal.so"
-  export GEOS_LIBRARY_PATH="$BUILD_DIR/.heroku/vendor/lib/libgeos_c.so"
-  # set path for runtime environmeht
-  set_env GDAL_LIBRARY_PATH "/app/.heroku/vendor/lib/libgdal.so"
-  set_env GEOS_LIBRARY_PATH "/app/.heroku/vendor/lib/libgeos_c.so"
-fi
@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-# Install Mercurial if it appears to be required.
-if [[ -f "requirements.txt" ]]; then
-	if (grep -Fiq "hg+" requirements.txt) then
-		/app/.heroku/python/bin/pip install  mercurial | cleanup | indent
-		mcount "steps.mercurial"
-	fi
-fi
@@ -14,7 +14,7 @@
 source "$BIN_DIR/utils"

 # Check that nltk was installed by pip, otherwise obviously not needed
-if sp-grep -s nltk; then
+if is_module_available 'nltk'; then
    puts-step "Downloading NLTK corpora…"

    nltk_packages_definition="$BUILD_DIR/nltk.txt"
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-set +e
-# Install dependencies with Pip.
-# shellcheck source=bin/utils
-source "$BIN_DIR/utils"
-
-if [ ! "$SKIP_PIP_INSTALL" ]; then
-
-  if [[ -f .heroku/python/requirements-declared.txt ]]; then
-
-    cp .heroku/python/requirements-declared.txt requirements-declared.txt
-
-
-    if ! pip-diff --stale requirements-declared.txt requirements.txt --exclude setuptools pip wheel > .heroku/python/requirements-stale.txt; then
-      mcount "failure.bad-requirements"
-    fi
-
-    rm -fr requirements-declared.txt
-
-    if [[ -s .heroku/python/requirements-stale.txt ]]; then
-      puts-step "Uninstalling stale dependencies"
-      /app/.heroku/python/bin/pip uninstall -r .heroku/python/requirements-stale.txt -y --exists-action=w --disable-pip-version-check | cleanup | indent
-    fi
-  fi
-
-fi
-set -e
@@ -69,7 +69,6 @@ if [ ! "$SKIP_PIPENV_INSTALL" ]; then

        else
            pipenv-to-pip Pipfile.lock > requirements.txt
-            "$BIN_DIR/steps/pip-uninstall"
            cp requirements.txt .heroku/python/requirements-declared.txt
            openssl dgst -sha256 Pipfile.lock > .heroku/python/Pipfile.lock.sha256

@@ -30,6 +30,9 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
                if [ "$PYTHON" = 3.8 ]; then
                    echo "$LATEST_38" > "$BUILD_DIR/runtime.txt"
                fi
+                if [ "$PYTHON" = 3.9 ]; then
+                    echo "$LATEST_39" > "$BUILD_DIR/runtime.txt"
+                fi
            fi


@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-# This script serves as the Pylibmc build step of the
-# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
-# compiler.
-#
-# A [buildpack](https://devcenter.heroku.com/articles/buildpacks) is an
-# adapter between a Python application and Heroku's runtime.
-#
-# This script is invoked by [`bin/compile`](/).
-
-if [[ "$STACK" != "cedar-14" ]]; then
-    # libmemcached is pre-installed in the stack image so there is no need to vendor it.
-    return 0
-fi
-
-# The location of the pre-compiled libmemcached binary.
-VENDORED_MEMCACHED="${VENDOR_URL}/libraries/vendor/libmemcache.tar.gz"
-
-# Syntax sugar.
-# shellcheck source=bin/utils
-source "$BIN_DIR/utils"
-
-# If pylibmc exists within requirements, use vendored libmemcached.
-if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
-
-  if [ ! -d ".heroku/vendor/lib/sasl2" ]; then
-    echo "-----> Noticed pylibmc. Bootstrapping libmemcached."
-    mkdir -p .heroku/vendor
-    # Download and extract libmemcached into target vendor directory.
-    curl "$VENDORED_MEMCACHED" -s | tar zxv -C .heroku/vendor &> /dev/null
-    mcount "steps.vendor.pylibmc"
-  fi
-
-  LIBMEMCACHED=$(pwd)/vendor
-  export LIBMEMCACHED
-fi
@@ -5,7 +5,7 @@ runtime-fixer runtime.txt
 PYTHON_VERSION=$(cat runtime.txt)

 # The location of the pre-compiled python binary.
-VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"
+VENDORED_PYTHON="${S3_BASE_URL}/${STACK}/runtimes/${PYTHON_VERSION}.tar.gz"

 SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
 SECURITY_UPDATE_PYPY="The PyPy project has released a security update! Please consider upgrading to"
@@ -16,6 +16,12 @@ PYTHON_2_EOL_UPDATE="Python 2 has reached it's community EOL. Upgrade your Pytho

 # check if runtime exists
 if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
+  if [[ "$PYTHON_VERSION" == $PY39* ]]; then
+    if [ "$PYTHON_VERSION" != "$LATEST_39" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_39"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
  if [[ "$PYTHON_VERSION" == $PY38* ]]; then
    # do things to alert the user of security release available
    if [ "$PYTHON_VERSION" != "$LATEST_38" ]; then
@@ -84,13 +90,13 @@ mcount "version.python.${PYTHON_VERSION}"

 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
    puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache"
-    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
+    rm -rf .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
 fi

 if [ -f .heroku/python-version ]; then
  if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then
      puts-step "Found $(cat .heroku/python-version), removing"
-      rm -fr .heroku/python
+      rm -rf .heroku/python
  else
    SKIP_INSTALL=1
  fi
@@ -160,7 +166,7 @@ fi
 # Instead, we use the pip wheel to install itself, using the method described here:
 # https://github.com/pypa/pip/issues/2351#issuecomment-69994524
 PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl"
-PIP_WHEEL_URL="https://lang-python.s3.amazonaws.com/common/${PIP_WHEEL_FILENAME}"
+PIP_WHEEL_URL="${S3_BASE_URL}/common/${PIP_WHEEL_FILENAME}"
 PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}"

 if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then
@@ -2,12 +2,13 @@
 shopt -s extglob
 shopt -s nullglob

-# The standard library.
-if [[ ! -f  /tmp/stdlib.sh ]]; then
-  curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v8/stdlib.sh > /tmp/stdlib.sh
-fi
-# shellcheck source=/dev/null
-source /tmp/stdlib.sh
+# This is necessary since this script is sometimes sourced from
+# subshells that don't have the variables from bin/compile.
+# Remove this once we no longer wrap all the things in `sub_env`.
+BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT_DIR=$(dirname "${BIN_DIR}")
+# shellcheck source=vendor/buildpack-stdlib_v8.sh
+source "${ROOT_DIR}/vendor/buildpack-stdlib_v8.sh"

 if [ "$(uname)" == Darwin ]; then
    sed() { command sed -l "$@"; }
@@ -95,3 +96,11 @@ python_sqlite3_check() {
  ( python2_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_2" ) \
    || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" )
 }
+
+is_module_available() {
+  # Returns 0 is the specified module exists, otherwise returns 1.
+  # Uses pkgutil rather than pkg_resources or pip's CLI, since pkgutil exists
+  # in the stdlib, and doesn't depend on the choice of package manager.
+  local module_name="${1}"
+  python -c "import sys, pkgutil; sys.exit(0 if pkgutil.find_loader('${module_name}') else 1)"
+}
@@ -12,16 +12,6 @@ old-platform() {
  fi
 }

-pylibmc-missing() {
-  if grep -qi 'fatal error: libmemcached/memcached.h: No such file or directory' "$WARNINGS_LOG"; then
-    echo
-    puts-warn "Hello! There was a problem with your build related to libmemcache."
-    puts-warn "The Python library 'pylibmc' must be explicitly specified in 'requirements.txt' in order to build correctly."
-    puts-warn "Once you do that, everything should work as expected.   -- Much Love, Heroku."
-    mcount 'warnings.libmemcache'
-  fi
-}
-
 scipy-included() {
  if grep -qi 'running setup.py install for scipy' "$WARNINGS_LOG"; then
    echo
@@ -56,11 +46,22 @@ six-included() {
  fi
 }

+gdal-missing() {
+  if grep -qi 'Could not find gdal-config' "$WARNINGS_LOG"; then
+    mcount 'warnings.gdal'
+    echo
+    puts-warn "Hello! Package installation failed since the GDAL library was not found."
+    puts-warn "For GDAL, GEOS and PROJ support, use the Geo buildpack alongside the Python buildpack:"
+    puts-warn "https://github.com/heroku/heroku-geo-buildpack"
+    puts-warn "  -- Much Love, Heroku."
+  fi
+}
+
 show-warnings() {
  old-platform
-  pylibmc-missing
  scipy-included
  distribute-included
  six-included
+  gdal-missing
 }

@@ -1,13 +1,21 @@
 [buildpack]
 name = "Python"

-  [publish.Ignore]
-  files = [
-    "test/",
-    ".gitignore",
-    ".dockerignore",
-    ".github/",
-    "Dockerfile",
-    "Pipfile",
-    "Pipfile.lock"
-  ]
+[publish.Ignore]
+files = [
+  ".git2gus/",
+  ".github/",
+  "builds/",
+  "spec/",
+  "test/",
+  ".dockerignore",
+  ".gitignore",
+  ".travis.yml",
+  "Gemfile",
+  "Gemfile.lock",
+  "hatchet.json",
+  "hatchet.lock",
+  "Makefile",
+  "Rakefile",
+  "requirements.txt",
+]
@@ -1,57 +1,80 @@
 # Python Buildpack Binaries

-## Building the Docker Images
-
-**After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack:
-
-    $ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
-    $ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-    $ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-
-## Using the Image
-
-You can e.g. `bash` into each of the images you built using their tag:
-
-    docker run --rm -ti heroku-python-build-cedar-14 bash
-    docker run --rm -ti heroku-python-build-heroku-16 bash
-    docker run --rm -ti heroku-python-build-heroku-18 bash
-
-You then have a shell where you can run `bob build`, `bob deploy`, and so forth. You can of course also invoke these programs directly with `docker run`:
-
-    docker run --rm -ti heroku-python-build-heroku-18 bob build runtimes/python-2.7.15
-
-In order to `bob deploy`, AWS credentials must be set up, as well as name and prefix of your custom S3 bucket (unless you're deploying to the Heroku production buckets that are pre-defined in each `Dockerfile`); see next section for details.
+The binaries for this buildpack are built in Docker containers based on the Heroku stack image.

 ## Configuration

-File `dockerenv.default` contains a list of required env vars; most of these have default values defined in `Dockerfile`. You can copy this file to a location outside the buildpack and modify it with the values you desire and pass its location with `--env-file`, or pass the env vars to `docker run` using `--env`.
+In order to publish binaries AWS credentials must be passed to the build container.
+If you are testing only the build (ie: `bob build`), these are optional.

-Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCKET`, `S3_PREFIX`, and `S3_REGION`. If you're building your own packages, you'll likely want to change `S3_BUCKET` and `S3_PREFIX` to match your info. Instead of setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` into that file, you may also pass them to `docker run` through the environment, or explicitly using `--env`, in order to prevent accidental commits of credentials.
+In addition, unless you are building the official binaries for Heroku (which use the defaults
+specified in each `Dockerfile`), you will need to override `S3_BUCKET` and `S3_PREFIX` to
+match your own S3 bucket/use case.

-### Passing AWS credentials to the container
+If you only need to set AWS credentials, you can do so by setting the environment variables
+`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` before calling the make commands.

-If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file.
+For example:

-#### Passing credentials explicitly
+```bash
+set +o history # Disable bash history
+export AWS_ACCESS_KEY_ID=...
+export AWS_SECRET_ACCESS_KEY=...
+set -o history # Re-enable bash history
+make ...
+```

-    docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash
+If you need to override the default S3 bucket, or would prefer not to use credentials via
+environment variables, then you need to instead use a Docker env file like so:

-#### Passing credentials through  the environment
+1. Copy the `builds/dockerenv.default` env file to a location outside the buildpack repository.
+2. Edit the new file, adding at a minimum the values for the variables
+   `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (see Docker
+   [env-file documentation](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file)).
+3. Pass the path of the file to the make commands using `ENV_FILE`. For example:

-The two environment variables `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY` are defined in `builds/dockerenv.default`, without values. This will cause Docker to "forward" values for these variables from the current environment, so you can pass them in:
+   ```bash
+   make ... ENV_FILE=~/.dockerenv.python-buildpack
+   ```

-    AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+## Launching an interactive build environment

-or
+To start an interactive version of the build environment (ideal for development) use the
+`buildenv` make target, passing in the desired `STACK` name. For example:

-    export AWS_ACCESS_KEY_ID=...
-    export AWS_SECRET_ACCESS_KEY=...
-    docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+```bash
+make buildenv STACK=heroku-18
+```

-#### Passing credentials through a separate env file
+This will create the builder docker image based on the latest image for that stack, and
+then start a bash shell where you can run `bob build`, `bob deploy`, and so forth.

-This method is the easiest for users who want to build packages in their own S3 bucket, as they will have to adjust the `S3_BUCKET` and `S3_PREFIX` environment variable values anyway from their default values.
+The `builds/` directory is bind-mounted into the running container, so local build formula
+changes will appear there immediately without the need to rebuild the image.

-For this method, it is important to keep the credentials file in a location outside the buildpack, so that your credentials aren't accidentally committed. Copy `builds/dockerenv.default` **to a safe location outside the buildpack directory**, and insert your values for `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY`.
+## Bulk deploying runtimes

-    docker run --rm -ti --env-file=../SOMEPATHOUTSIDE/s3.env heroku-python-build-heroku-18 bash
+When a new Python version is released, binaries have to be generated for multiple stacks.
+To automate this, use the `deploy-runtimes` make target, which will ensure the builder
+image is up to date, and then run `bob deploy` for each runtime-stack combination.
+
+The build formula name(s) are passed using `RUNTIMES`, like so:
+
+```bash
+make deploy-runtimes RUNTIMES='python-X.Y.Z'
+```
+
+By default this will deploy to all supported stacks (see `STACKS` in `Makefile`),
+but this can be overridden using `STACKS`:
+
+```bash
+make deploy-runtimes RUNTIMES='python-X.Y.Z' STACKS='heroku-16 heroku-18'
+```
+
+Multiple runtimes can also be specified (useful for when adding a new stack), like so:
+
+```bash
+make deploy-runtimes RUNTIMES='python-A.B.C python-X.Y.Z' STACKS='heroku-20'
+```
+
+Note: Both `RUNTIMES` and `STACKS` are space delimited.
@@ -1,15 +0,0 @@
-FROM heroku/cedar:14
-
-WORKDIR /app
-ENV WORKSPACE_DIR="/app/builds" \
-    S3_BUCKET="lang-python" \
-    S3_PREFIX="cedar-14/" \
-    DEBIAN_FRONTEND=noninteractive \
-    STACK="cedar-14"
-
-RUN apt-get update && apt-get install -y python-pip libsqlite3-dev realpath && rm -rf /var/lib/apt/lists/*
-
-COPY requirements.txt /app/
-RUN pip install -r /app/requirements.txt
-
-COPY . /app
@@ -1,5 +1,8 @@
+# Since no values are specified here, these variables will be read from the environment at run time:
+# https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file
 AWS_ACCESS_KEY_ID
 AWS_SECRET_ACCESS_KEY
-S3_BUCKET
-S3_PREFIX
-S3_REGION
+
+# Uncomment these if you need to override the default S3 bucket and/or path prefixes.
+# S3_BUCKET
+# S3_PREFIX
@@ -1,15 +1,20 @@
 FROM heroku/heroku:16-build

-WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
-    S3_BUCKET="lang-python" \
+    S3_BUCKET="heroku-buildpack-python" \
    S3_PREFIX="heroku-16/" \
-    DEBIAN_FRONTEND=noninteractive \
    STACK="heroku-16"

-RUN apt-get update && apt-get install -y python-pip libsqlite3-dev && rm -rf /var/lib/apt/lists/*
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        python3-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app

 COPY requirements.txt /app/
-RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt

 COPY . /app
@@ -1,15 +1,20 @@
 FROM heroku/heroku:18-build

-WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
-    S3_BUCKET="lang-python" \
+    S3_BUCKET="heroku-buildpack-python" \
    S3_PREFIX="heroku-18/" \
-    DEBIAN_FRONTEND=noninteractive \
    STACK="heroku-18"

-RUN apt-get update && apt-get install --no-install-recommends -y python-pip-whl=9.0.1-2 python-pip=9.0.1-2 python-setuptools python-wheel libsqlite3-dev && rm -rf /var/lib/apt/lists/*
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        python3-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app

 COPY requirements.txt /app/
-RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt

 COPY . /app
@@ -0,0 +1,20 @@
+FROM heroku/heroku:20-build
+
+ENV WORKSPACE_DIR="/app/builds" \
+    S3_BUCKET="heroku-buildpack-python" \
+    S3_PREFIX="heroku-20/" \
+    STACK="heroku-20"
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        python3-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt /app/
+RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+
+COPY . /app
@@ -19,4 +19,4 @@ make install

 # Cleanup
 cd ..
-rm -fr sqlite
+rm -rf sqlite
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building gdal…"
-
-VERSION="2.2.1"
-SOURCE_TARBALL="http://download.osgeo.org/gdal/${VERSION}/gdal-${VERSION}.tar.gz"
-
-curl -L $SOURCE_TARBALL | tar zx
-
-pushd "gdal-${VERSION}"
-./configure --prefix=$OUT_PREFIX --enable-static=no  &&
-make
-make install
-
-# Cleanup
-popd
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building geos…"
-
-VERSION=3.6.2
-
-SOURCE_TARBALL="http://download.osgeo.org/geos/geos-${VERSION}.tar.bz2"
-
-curl -L $SOURCE_TARBALL | tar xj
-
-pushd "geos-${VERSION}"
-./configure --prefix=$OUT_PREFIX --enable-static=no &&
-make
-make install
-
-# Cleanup
-popd
@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-if [[ $S3_PREFIX != "cedar-14" ]]; then
-    echo "libmemcached only needs to be built for cedar-14, since newer stacks include it in the base image"
-    exit 1
-fi
-
-# fail hard
-set -o pipefail
-# fail harder
-set -eux
-
-DEFAULT_VERSION="1.0.18"
-dep_version=${VERSION:-$DEFAULT_VERSION}
-dep_dirname=libmemcached-${dep_version}
-dep_archive_name=${dep_dirname}.tar.gz
-dep_url=https://launchpad.net/libmemcached/1.0/${dep_version}/+download/${dep_archive_name}
-
-# SASL Support.
-echo "-----> Building cyrus-sasl 2.1.26…"
-
-curl -LO ftp://ftp.cyrusimap.org/cyrus-sasl/cyrus-sasl-2.1.26.tar.gz
-# FTP doesn't play well with piping into tar xz
-tar xzf cyrus-sasl-2.1.26.tar.gz
-
-pushd cyrus-sasl-2.1.26
-./configure --prefix=${OUT_PREFIX} --with-plugindir=${OUT_PREFIX}lib/sasl2 --with-configdir=${OUT_PREFIX}lib/sasl2
-
-make -s -j 9
-make install -s
-popd
-
-echo "-----> Building libmemcached ${dep_version}…"
-
-curl -L ${dep_url} | tar xz
-pushd ${dep_dirname}
-CPPFLAGS=-I${OUT_PREFIX}/include LDFLAGS=-L${OUT_PREFIX}/lib ./configure --prefix=${OUT_PREFIX} --without-memcached
-make -s -j 9
-make install -s
-popd
-
-echo "-----> Done."
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building gdal…"
-
-VERSION=4.9.3
-SOURCE_TARBALL="http://download.osgeo.org/proj/proj-${VERSION}.tar.gz"
-
-curl -L $SOURCE_TARBALL | tar zx
-
-pushd "proj-${VERSION}"
-./configure --prefix=$OUT_PREFIX --enable-static=no
-make
-make install
-
-# Cleanup
-popd
@@ -13,11 +13,11 @@ dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full ver
 dep_package=${dep_name}-v${dep_version} # it's always "pypy2-…"
 dep_dirname=${dep_package}-linux64
 dep_archive_name=${dep_dirname}.tar.bz2
-dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+dep_url="https://downloads.python.org/pypy/${dep_archive_name}"

 echo "Building PyPy…"
 echo "${dep_url}"

-curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+curl -fL "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name

 ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python"
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy2.7
@@ -13,11 +13,11 @@ dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full ver
 dep_package=${dep_name}${dep_version_prefix:-}-v${dep_version}${dep_version_suffix:-}
 dep_dirname=${dep_package}-linux64
 dep_archive_name=${dep_dirname}.tar.bz2
-dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+dep_url="https://downloads.python.org/pypy/${dep_archive_name}"

 echo "Building PyPy3…"
 echo "${dep_url}"

-curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+curl -fL "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name

 ln "$OUT_PREFIX/bin/pypy3" "$OUT_PREFIX/bin/python"
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy3.6
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -1,3 +1,6 @@
-docopt==0.6.2
-bob-builder==0.0.18
+# Dependencies for generating/publishing Python binaries.
+bob-builder==0.0.19
+
+# Sub-dependencies of bob-builder.
 boto==2.49.0
+docopt==0.6.2
@@ -0,0 +1 @@
+gdal
@@ -1 +0,0 @@
-django
@@ -1 +0,0 @@
-python-3.6.6
@@ -6,4 +6,4 @@ verify_ssl = true
 requests = "*"

 [requires]
-python_full_version = "3.6.3"
+python_full_version = "3.7.8"
@@ -1,24 +1,11 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "22a052f4d1cfe6518b2f236fe45c3208c587a9ab1323bdd390632e27278b541e"
-        },
-        "host-environment-markers": {
-            "implementation_name": "cpython",
-            "implementation_version": "3.6.3",
-            "os_name": "posix",
-            "platform_machine": "x86_64",
-            "platform_python_implementation": "CPython",
-            "platform_release": "16.7.0",
-            "platform_system": "Darwin",
-            "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64",
-            "python_full_version": "3.6.3",
-            "python_version": "3.6",
-            "sys_platform": "darwin"
+            "sha256": "8a36860f0f9cb55716222098062cea5c5e0f8127cafb9d0c694de327bac9fbc0"
        },
        "pipfile-spec": 6,
        "requires": {
-            "python_full_version": "3.6.3"
+            "python_full_version": "3.7.8"
        },
        "sources": [
            {
@@ -1 +1 @@
-pypy2.7-7.3.1
+pypy2.7-7.3.2
@@ -1 +1 @@
-pypy2.7-7.2.0
+pypy2.7-7.3.1
@@ -1 +1 @@
-pypy3.6-7.3.1
+pypy3.6-7.3.2
@@ -1 +1 @@
-pypy3.6-7.2.0
+pypy3.6-7.3.1
@@ -1 +0,0 @@
-requests
@@ -1 +0,0 @@
-flask
@@ -1 +0,0 @@
-flask
@@ -1 +1 @@
-python-3.6.8
+python-3.6.12
@@ -1 +0,0 @@
-requests
@@ -1 +1 @@
-python-3.6.7
+python-3.6.11
@@ -1 +1 @@
-python-3.7.2
+python-3.7.9
@@ -1 +0,0 @@
-requests
@@ -1 +1 @@
-python-3.7.1
+python-3.7.8
@@ -1 +1 @@
-python-3.8.2
+python-3.8.6
@@ -1 +0,0 @@
-requests
@@ -1 +1 @@
-python-3.8.0
+python-3.8.5
@@ -0,0 +1 @@
+python-3.9.0
@@ -0,0 +1 @@
+python-3.9.0
@@ -1 +0,0 @@
-flask
@@ -0,0 +1 @@
+hg+https://www.mercurial-scm.org/repo/python-hglib/#egg=python-hglib
@@ -17,21 +17,18 @@ testCollectstatic() {
  assertCaptured "collectstatic"
 }

-testGEOS() {
+testBuildWithGeoLibrariesWarning() {
  local env_dir="$(mktmpdir)"
  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
-  compile 'geos' '' "${env_dir}"
-  assertCaptured "geos"
-  assertCapturedSuccess
+  compile 'gdal' '' "${env_dir}"
+  assertCaptured " !     The Python buildpack's legacy BUILD_WITH_GEO_LIBRARIES functonality"
+  assertCapturedError
 }

-testGEOSDeprecation() {
-  local env_dir="$(mktmpdir)"
-  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
-  compile 'geos' '' "${env_dir}"
-  assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated.
- !     An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
-  assertCapturedSuccess
+testGDALWarning() {
+  compile 'gdal'
+  assertCaptured " !     Hello! Package installation failed since the GDAL library was not found."
+  assertCapturedError
 }

 testNLTK() {
@@ -43,7 +40,9 @@ testNLTK() {
  echo 'ignore::RuntimeWarning' > "${env_dir}/PYTHONWARNINGS"
  compile 'nltk' '' "${env_dir}"
  assertCaptured "[nltk_data] Downloading package city_database" "STD_ERR"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since the NLTK downloader outputs all
+  # progress/status messages to stderr (W-8146040).
+  assertCapturedSuccessWithStdErr
 }

 testPsycopg2() {
@@ -53,17 +52,16 @@ testPsycopg2() {
 }

 testPysqlite() {
+  # pysqlite does not support Python 3 (since the sqlite3 stdlib can be used there),
+  # so we have to test with Python 2, which we've not made available for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
  compile "pysqlite"
  assertCaptured "pysqlite"
  assertCapturedSuccess
 }

-testSqliteInstall() {
-  compile "pythonDefault"
-  assertNotCaptured "Sqlite3 failed to install."
-  assertCapturedSuccess
-}
-
 testCffi() {
  compile "cffi"
  assertCaptured "cffi"
@@ -76,6 +74,12 @@ testPylibmc() {
  assertCapturedSuccess
 }

+testMercurial() {
+  compile "requirements-mercurial"
+  assertCaptured "Cloning hg"
+  assertCapturedSuccess
+}
+
 pushd $(dirname 0) >/dev/null
 popd >/dev/null

@@ -33,7 +33,9 @@ testStackChange() {
 testSetupPy() {
  compile "setup-py"
  assertCaptured "maya"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }

 testStandardRequirements() {
@@ -45,29 +47,44 @@ testStandardRequirements() {
 testPipenv() {
  compile "pipenv"
  assertCaptured "Installing pip 9.0.2, setuptools 47.1.1 and wheel 0.34.2"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }

 testPipenvLock() {
  compile "pipenv-lock"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }

-testPipenvVersion() {
+testPipenvPythonVersion3_6() {
  compile "pipenv-version"
-  assertCaptured $DEFAULT_PYTHON_VERSION
-  assertCapturedSuccess
+  assertCaptured "Installing ${LATEST_36}"
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }

-testPipenvVersion2() {
+testPipenvPythonVersion2_7() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
  compile "pipenv-version2"
-  assertCaptured $LATEST_27
-  assertCapturedSuccess
+  assertCaptured "Installing ${LATEST_27}"
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
-testPipenvFullVersion() {
+
+testPipenvPythonFullVersion() {
  compile "pipenv-full-version"
-  assertCaptured "3.6.3"
-  assertCapturedSuccess
+  assertCaptured "3.7.8"
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }

 testNoRequirements() {
@@ -95,6 +112,7 @@ testHooks() {
  local expected_env_vars=(
    _
    BIN_DIR
+    BPLOG_PREFIX
    BUILD_DIR
    BUILDPACK_LOG_FILE
    CACHE_DIR
@@ -116,9 +134,8 @@ testHooks() {
    SHLVL
    SOME_APP_CONFIG_VAR
    STACK
-    VENDOR_URL
  )
-  if [[ "${STACK}" == "cedar-14" || "${STACK}" == "heroku-16" ]]; then
+  if [[ "${STACK}" == "heroku-16" ]]; then
    # Remove "OLDPWD" from expected_env_vars since for bash <4.4 it's not exported to subshells:
    # https://github.com/heroku/heroku-buildpack-python/pull/1011#issuecomment-665117835
    read -ra expected_env_vars <<< "${expected_env_vars[@]/OLDPWD/}"
@@ -4,55 +4,78 @@
 # shellcheck source=bin/default_pythons
 source "bin/default_pythons"

-testPythonDefault() {
-  updateVersion "pythonDefault" $DEFAULT_PYTHON_VERSION
-  compile "pythonDefault"
-  assertCaptured $DEFAULT_PYTHON_VERSION
+testPythonVersionUnspecified() {
+  compile "python_version_unspecified"
+  assertCaptured "Installing ${DEFAULT_PYTHON_VERSION}"
  assertNotCaptured "security update"
  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
  assertCaptured "Installing SQLite3"
  assertCapturedSuccess
 }

-testPython2() {
-    updateVersion "python2" $LATEST_27
-    echo $LATEST_27 > "runtime.txt"
-    compile "python2"
-    assertCaptured $LATEST_27
-    assertCaptured "python-2-7-eol-faq";
-    assertNotCaptured "security update"
-    assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+testPython2_7() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
+  compile "python2"
+  assertCaptured "Installing ${LATEST_27}"
+  assertCaptured "python-2-7-eol-faq";
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }

-testPython2_warn() {
-    compile "python2_warn"
-    assertCaptured "python-2.7.15"
-    assertCaptured "python-2-7-eol-faq";
-    assertCaptured "Only the latest version"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+testPython2_7_warn() {
+  # Python 2.7 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
+  compile "python2_warn"
+  assertCaptured "Installing python-2.7.15"
+  assertCaptured "python-2-7-eol-faq";
+  assertCaptured "Only the latest version"
+  assertCaptured "${LATEST_27}"
+  assertCapturedSuccess
 }

 testPython3_4() {
+  # Python 3.4 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
  compile "python3_4"
-  assertCaptured $LATEST_34
+  assertCaptured "Installing ${LATEST_34}"
  assertNotCaptured "security update"
  assertCaptured "Installing pip 19.1.1, setuptools 43.0.0 and wheel 0.33.6"
-  assertCapturedSuccess
+  assertCaptured "Installing SQLite3"
+  # Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
+  # and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
+  assertCapturedSuccessWithStdErr
 }

 testPython3_4_warn() {
+  # Python 3.4 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
  compile "python3_4_warn"
-  assertCaptured "python-3.4.9"
+  assertCaptured "Installing python-3.4.9"
  assertCaptured "security update!"
-  assertCapturedSuccess
+  assertCaptured "${LATEST_34}"
+  # Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
+  # and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
+  assertCapturedSuccessWithStdErr
 }

 testPython3_5() {
+  # Python 3.5 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
  compile "python3_5"
-  assertCaptured $LATEST_35
+  assertCaptured "Installing ${LATEST_35}"
  assertNotCaptured "security update"
  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
  assertCaptured "Installing SQLite3"
@@ -60,16 +83,20 @@ testPython3_5() {
 }

 testPython3_5_warn() {
+  # Python 3.5 is EOL, so it has not been built for Heroku-20.
+  if [[ $STACK == "heroku-20" ]]; then
+    return
+  fi
  compile "python3_5_warn"
-  assertCaptured "python-3.5.6"
+  assertCaptured "Installing python-3.5.6"
  assertCaptured "security update!"
+  assertCaptured "${LATEST_35}"
  assertCapturedSuccess
 }

 testPython3_6() {
-  updateVersion "python3_6" $LATEST_36
  compile "python3_6"
-  assertCaptured $LATEST_36
+  assertCaptured "Installing ${LATEST_36}"
  assertNotCaptured "security update"
  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
  assertCaptured "Installing SQLite3"
@@ -78,62 +105,65 @@ testPython3_6() {

 testPython3_6_warn() {
  compile "python3_6_warn"
-  assertCaptured "python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
  assertCaptured "security update!"
-  assertCaptured "Installing SQLite3"
+  assertCaptured "${LATEST_36}"
  assertCapturedSuccess
 }

 testPython3_7() {
-  updateVersion "python3_7" $LATEST_37
  compile "python3_7"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertNotCaptured "security update"
-    assertCaptured $LATEST_37
-    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing ${LATEST_37}"
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }

 testPython3_7_warn() {
  compile "python3_7_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "python-3.7.1"
-    assertCaptured "security update!"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing python-3.7.8"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_37}"
+  assertCapturedSuccess
 }

 testPython3_8() {
-  updateVersion "python3_8" $LATEST_38
  compile "python3_8"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertNotCaptured "security update"
-    assertCaptured $LATEST_38
-    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing ${LATEST_38}"
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }

 testPython3_8_warn() {
  compile "python3_8_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "python-3.8.0"
-    assertCaptured "security update!"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+  assertCaptured "Installing python-3.8.5"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_38}"
+  assertCapturedSuccess
+}
+
+testPython3_9() {
+  compile "python3_9"
+  assertCaptured "Installing ${LATEST_39}"
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
+}
+
+testPython3_9_warn() {
+  # Can't test the version warning until there is at least one old version of Python 3.9.
+  if [[ "${LATEST_39}" = "python-3.9.0" ]]; then
+    return
  fi
+  compile "python3_9_warn"
+  assertCaptured "Installing python-3.9.0"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_39}"
+  assertCapturedSuccess
 }

 testPythonVersionInvalid() {
@@ -145,67 +175,58 @@ testPythonVersionInvalid() {

 testPypy3_6() {
  compile "pypy3_6"
-  assertCaptured "Installing pypy"
+  assertCaptured "Installing ${LATEST_PYPY_36}"
  assertNotCaptured "security update"
-  assertCaptured "$LATEST_PYPY_36"
  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
  assertCapturedSuccess
 }

 testPypy3_6_warn() {
  compile "pypy3_6_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "Installing pypy"
-    assertCaptured "security update!"
-    assertCaptured "$LATEST_PYPY_36"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing pypy3.6-7.3.1"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_PYPY_36}"
+  assertCapturedSuccess
 }

 testPypy2_7() {
  compile "pypy2_7"
-  assertCaptured "Installing pypy"
+  assertCaptured "Installing ${LATEST_PYPY_27}"
  assertNotCaptured "security update"
-  assertCaptured "$LATEST_PYPY_27"
  assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
  assertCapturedSuccess
 }

 testPypy2_7_warn() {
  compile "pypy2_7_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "Installing pypy"
-    assertCaptured "security update!"
-    assertCaptured "$LATEST_PYPY_27"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing pypy2.7-7.3.1"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_PYPY_27}"
+  assertCapturedSuccess
 }

 testStickyPythonVersion() {
  local cache_dir="$(mktmpdir)"
  compile "python3_6_warn" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
  assertCapturedSuccess
-  compile "no-runtime-txt" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  compile "python_version_unspecified" "$cache_dir"
+  assertNotCaptured "Installing python"
+  assertCaptured "security update!"
  assertCapturedSuccess
  # Whilst this file seems like an implementation detail (so something that should
  # not be tested), we must guarantee the filename remains consistent for backwards
  # compatibility across buildpack versions for already-built apps.
-  assertFile "python-3.6.7" ".heroku/python-version"
+  assertFile "python-3.6.11" ".heroku/python-version"
 }

 testPythonVersionChange() {
  local cache_dir="$(mktmpdir)"
  compile "python3_6_warn" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  assertCaptured "Installing python-3.6.11"
  assertCapturedSuccess
  compile "python3_6" "$cache_dir"
-  assertCaptured "Found python-3.6.7, removing"
+  assertCaptured "Found python-3.6.11, removing"
  assertCapturedSuccess
 }

@@ -56,11 +56,6 @@ resetCapture()
  unset rtrn # deprecated
 }

-updateVersion()
-{
-  echo "$2" > "test/fixtures/${1}/runtime.txt"
-}
-
 assertCapturedEquals()
 {
  assertEquals "$@" "$(cat ${STD_OUT})"
@@ -85,12 +80,20 @@ assertNotCaptured()
 assertCapturedSuccess()
 {
  assertEquals "Captured exit code -" "0" "${RETURN}"
-  # assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
+  assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})"

-  if [ $RETURN -ne 0 -a -z "$(cat ${STD_ERR})" ]; then
-    # Failing exit code but stderr was empty. Display stdout to help debugging.
-    cat $STD_OUT
-    echo
+  if [ $RETURN -ne 0 -o -n "$(cat ${STD_ERR})" ]; then
+    debug
+  fi
+}
+
+assertCapturedSuccessWithStdErr()
+{
+  assertEquals "Captured exit code -" "0" "${RETURN}"
+  assertNotEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
+
+  if [ ${RETURN} -ne 0 ]; then
+    debug
  fi
 }

@@ -151,9 +154,14 @@ _assertContains()

 debug()
 {
-  cat $STD_OUT
-  echo '^^^^^^'
-  cat $STD_ERR
+  echo
+  echo '### STD_OUT ###'
+  cat "${STD_OUT}"
+  echo
+  echo '### STD_ERR ###'
+  cat "${STD_ERR}"
+  echo
+  echo
 }

 assertContains()
@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+
+# From:
+# https://raw.githubusercontent.com/heroku/buildpack-stdlib/v8/stdlib.sh
+
+# Buildpack defaults
+# ---------------
+
+export BUILDPACK_LOG_FILE="${BUILDPACK_LOG_FILE:-/dev/null}"
+
+# Standard Output
+# ---------------
+
+# Buildpack Steps.
+puts_step() {
+  if [[ "$*" == "-" ]]; then
+    read -r output
+  else
+    output=$*
+  fi
+  echo -e "\\e[1m\\e[36m=== $output\\e[0m"
+  unset output
+}
+
+# Buildpack Error.
+puts_error() {
+  if [[ "$*" == "-" ]]; then
+    read -r output
+  else
+    output=$*
+  fi
+  echo -e "\\e[1m\\e[31m=!= $output\\e[0m"
+}
+
+# Buildpack Warning.
+puts_warn() {
+  if [[ "$*" == "-" ]]; then
+    read -r output
+  else
+    output=$*
+  fi
+  echo -e "\\e[1m\\e[33m=!= $output\\e[0m"
+}
+
+# Is verbose set?
+is_verbose() {
+  if [[ -n $BUILDPACK_VERBOSE ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+# Buildpack Verbose.
+puts_verbose() {
+  if is_verbose; then
+    if [[ "$*" == "-" ]]; then
+      read -r output
+    else
+      output=$*
+    fi
+    echo "$output"
+    unset output
+  fi
+}
+
+# Buildpack Utilities
+# -------------------
+
+# Usage: $ set-env key value
+# NOTICE: Expects PROFILE_PATH & EXPORT_PATH to be set!
+set_env() {
+  # TODO: automatically create profile path directory if it doesn't exist.
+  echo "export $1=$2" >> "$PROFILE_PATH"
+  echo "export $1=$2" >> "$EXPORT_PATH"
+}
+
+# Usage: $ set-default-env key value
+# NOTICE: Expects PROFILE_PATH & EXPORT_PATH to be set!
+set_default_env() {
+  echo "export $1=\${$1:-$2}" >> "$PROFILE_PATH"
+  echo "export $1=\${$1:-$2}" >> "$EXPORT_PATH"
+}
+
+# Usage: $ un-set-env key
+# NOTICE: Expects PROFILE_PATH to be set!
+un_set_env() {
+  echo "unset $1" >> "$PROFILE_PATH"
+}
+
+# Usage: $ _env-blacklist pattern
+# Outputs a regex of default blacklist env vars.
+_env_blacklist() {
+  local regex=${1:-''}
+  if [ -n "$regex" ]; then
+    regex="|$regex"
+  fi
+  echo "^(PATH|GIT_DIR|CPATH|CPPATH|LD_PRELOAD|LIBRARY_PATH$regex)$"
+}
+
+# Usage: $ export-env ENV_DIR WHITELIST BLACKLIST
+# Exports the environment variables defined in the given directory.
+export_env() {
+  local env_dir=${1:-$ENV_DIR}
+  local whitelist=${2:-''}
+  local blacklist
+  blacklist="$(_env_blacklist "$3")"
+  if [ -d "$env_dir" ]; then
+    # Environment variable names won't contain characters affected by:
+    # shellcheck disable=SC2045
+    for e in $(ls "$env_dir"); do
+      echo "$e" | grep -E "$whitelist" | grep -qvE "$blacklist" &&
+      export "$e=$(cat "$env_dir/$e")"
+      :
+    done
+  fi
+}
+
+# Usage: $ sub-env command
+# Runs a subshell of specified command with user-provided config.
+# NOTICE: Expects ENV_DIR to be set. WHITELIST & BLACKLIST are optional.
+# Examples:
+#    WHITELIST=${2:-''}
+#    BLACKLIST=${3:-'^(GIT_DIR|PYTHONHOME|LD_LIBRARY_PATH|LIBRARY_PATH|PATH)$'}
+sub_env() {
+  (
+    # TODO: Fix https://github.com/heroku/buildpack-stdlib/issues/37
+    # shellcheck disable=SC2153
+    export_env "$ENV_DIR" "$WHITELIST" "$BLACKLIST"
+
+    "$@"
+  )
+}
+
+# Logging
+# -------
+
+# Notice: These functions expect BPLOG_PREFIX and BUILDPACK_LOG_FILE to be defined (BUILDPACK_LOG_FILE can point to /dev/null if not provided by the buildpack).
+# Example: BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}; BPLOG_PREFIX="buildpack.go"
+
+# Returns now, in milleseconds. Useful for logging.
+# Example: $ let start=$(nowms); sleep 30; mtime "glide.install.time" "${start}"
+nowms() {
+  date +%s%3N
+}
+
+# Log arbitrary data to the logfile (e.g. a packaging file).
+# Usage: $ bplog "$(<${vendorJSON})
+bplog() {
+  echo -n "${@}" | awk 'BEGIN {printf "msg=\""; f="%s"} {gsub(/"/, "\\\"", $0); printf f, $0} {if (NR == 1) f="\\n%s" } END { print "\"" }' >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Measures time elapsed for a specific build step.
+# Usage: $ let start=$(nowms); mtime "glide.install.time" "${start}"
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#distributions-measure
+mtime() {
+  local key="${BPLOG_PREFIX}.${1}"
+  local start="${2}"
+  local end="${3:-$(nowms)}"
+  echo "${key} ${start} ${end}" | awk '{ printf "measure#%s=%.3f\n", $1, ($3 - $2)/1000 }' >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Logs a count for a specific built step.
+# Usage: $ mcount "tool.govendor"
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#counting-count
+mcount() {
+  local k="${BPLOG_PREFIX}.${1}"
+  local v="${2:-1}"
+  echo "count#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Logs a measure for a specific build step.
+# Usage: $ mmeasure "tool.installed_dependencies" 42
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#distributions-measure
+mmeasure() {
+  local k="${BPLOG_PREFIX}.${1}"
+  local v="${2}"
+  echo "measure#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Logs a unuique measurement build step.
+# Usage: $ munique "versions.count" 2.7.13
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#uniques-unique
+munique() {
+  local k="${BPLOG_PREFIX}.${1}"
+  local v="${2}"
+  echo "unique#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Measures when an exit path to the buildpack is reached, given a name, then exits 1.
+# Usage: $ mcount-exi "binExists"
+mcount_exit() {
+  mcount "error.${1}"
+  exit 1
+}
@@ -1,581 +0,0 @@
-"""Pythonic command-line interface parser that will make you smile.
-
- * http://docopt.org
- * Repository and issue-tracker: https://github.com/docopt/docopt
- * Licensed under terms of MIT license (see LICENSE-MIT)
- * Copyright (c) 2013 Vladimir Keleshev, vladimir@keleshev.com
-
-"""
-import sys
-import re
-
-
-__all__ = ['docopt']
-__version__ = '0.6.1'
-
-
-class DocoptLanguageError(Exception):
-
-    """Error in construction of usage-message by developer."""
-
-
-class DocoptExit(SystemExit):
-
-    """Exit in case user invoked program with incorrect arguments."""
-
-    usage = ''
-
-    def __init__(self, message=''):
-        SystemExit.__init__(self, (message + '\n' + self.usage).strip())
-
-
-class Pattern(object):
-
-    def __eq__(self, other):
-        return repr(self) == repr(other)
-
-    def __hash__(self):
-        return hash(repr(self))
-
-    def fix(self):
-        self.fix_identities()
-        self.fix_repeating_arguments()
-        return self
-
-    def fix_identities(self, uniq=None):
-        """Make pattern-tree tips point to same object if they are equal."""
-        if not hasattr(self, 'children'):
-            return self
-        uniq = list(set(self.flat())) if uniq is None else uniq
-        for i, child in enumerate(self.children):
-            if not hasattr(child, 'children'):
-                assert child in uniq
-                self.children[i] = uniq[uniq.index(child)]
-            else:
-                child.fix_identities(uniq)
-
-    def fix_repeating_arguments(self):
-        """Fix elements that should accumulate/increment values."""
-        either = [list(child.children) for child in transform(self).children]
-        for case in either:
-            for e in [child for child in case if case.count(child) > 1]:
-                if type(e) is Argument or type(e) is Option and e.argcount:
-                    if e.value is None:
-                        e.value = []
-                    elif type(e.value) is not list:
-                        e.value = e.value.split()
-                if type(e) is Command or type(e) is Option and e.argcount == 0:
-                    e.value = 0
-        return self
-
-
-def transform(pattern):
-    """Expand pattern into an (almost) equivalent one, but with single Either.
-
-    Example: ((-a | -b) (-c | -d)) => (-a -c | -a -d | -b -c | -b -d)
-    Quirks: [-a] => (-a), (-a...) => (-a -a)
-
-    """
-    result = []
-    groups = [[pattern]]
-    while groups:
-        children = groups.pop(0)
-        parents = [Required, Optional, OptionsShortcut, Either, OneOrMore]
-        if any(t in map(type, children) for t in parents):
-            child = [c for c in children if type(c) in parents][0]
-            children.remove(child)
-            if type(child) is Either:
-                for c in child.children:
-                    groups.append([c] + children)
-            elif type(child) is OneOrMore:
-                groups.append(child.children * 2 + children)
-            else:
-                groups.append(child.children + children)
-        else:
-            result.append(children)
-    return Either(*[Required(*e) for e in result])
-
-
-class LeafPattern(Pattern):
-
-    """Leaf/terminal node of a pattern tree."""
-
-    def __init__(self, name, value=None):
-        self.name, self.value = name, value
-
-    def __repr__(self):
-        return '%s(%r, %r)' % (self.__class__.__name__, self.name, self.value)
-
-    def flat(self, *types):
-        return [self] if not types or type(self) in types else []
-
-    def match(self, left, collected=None):
-        collected = [] if collected is None else collected
-        pos, match = self.single_match(left)
-        if match is None:
-            return False, left, collected
-        left_ = left[:pos] + left[pos + 1:]
-        same_name = [a for a in collected if a.name == self.name]
-        if type(self.value) in (int, list):
-            if type(self.value) is int:
-                increment = 1
-            else:
-                increment = ([match.value] if type(match.value) is str
-                             else match.value)
-            if not same_name:
-                match.value = increment
-                return True, left_, collected + [match]
-            same_name[0].value += increment
-            return True, left_, collected
-        return True, left_, collected + [match]
-
-
-class BranchPattern(Pattern):
-
-    """Branch/inner node of a pattern tree."""
-
-    def __init__(self, *children):
-        self.children = list(children)
-
-    def __repr__(self):
-        return '%s(%s)' % (self.__class__.__name__,
-                           ', '.join(repr(a) for a in self.children))
-
-    def flat(self, *types):
-        if type(self) in types:
-            return [self]
-        return sum([child.flat(*types) for child in self.children], [])
-
-
-class Argument(LeafPattern):
-
-    def single_match(self, left):
-        for n, pattern in enumerate(left):
-            if type(pattern) is Argument:
-                return n, Argument(self.name, pattern.value)
-        return None, None
-
-    @classmethod
-    def parse(class_, source):
-        name = re.findall('(<\S*?>)', source)[0]
-        value = re.findall('\[default: (.*)\]', source, flags=re.I)
-        return class_(name, value[0] if value else None)
-
-
-class Command(Argument):
-
-    def __init__(self, name, value=False):
-        self.name, self.value = name, value
-
-    def single_match(self, left):
-        for n, pattern in enumerate(left):
-            if type(pattern) is Argument:
-                if pattern.value == self.name:
-                    return n, Command(self.name, True)
-                else:
-                    break
-        return None, None
-
-
-class Option(LeafPattern):
-
-    def __init__(self, short=None, long=None, argcount=0, value=False):
-        assert argcount in (0, 1)
-        self.short, self.long, self.argcount = short, long, argcount
-        self.value = None if value is False and argcount else value
-
-    @classmethod
-    def parse(class_, option_description):
-        short, long, argcount, value = None, None, 0, False
-        options, _, description = option_description.strip().partition('  ')
-        options = options.replace(',', ' ').replace('=', ' ')
-        for s in options.split():
-            if s.startswith('--'):
-                long = s
-            elif s.startswith('-'):
-                short = s
-            else:
-                argcount = 1
-        if argcount:
-            matched = re.findall('\[default: (.*)\]', description, flags=re.I)
-            value = matched[0] if matched else None
-        return class_(short, long, argcount, value)
-
-    def single_match(self, left):
-        for n, pattern in enumerate(left):
-            if self.name == pattern.name:
-                return n, pattern
-        return None, None
-
-    @property
-    def name(self):
-        return self.long or self.short
-
-    def __repr__(self):
-        return 'Option(%r, %r, %r, %r)' % (self.short, self.long,
-                                           self.argcount, self.value)
-
-
-class Required(BranchPattern):
-
-    def match(self, left, collected=None):
-        collected = [] if collected is None else collected
-        l = left
-        c = collected
-        for pattern in self.children:
-            matched, l, c = pattern.match(l, c)
-            if not matched:
-                return False, left, collected
-        return True, l, c
-
-
-class Optional(BranchPattern):
-
-    def match(self, left, collected=None):
-        collected = [] if collected is None else collected
-        for pattern in self.children:
-            m, left, collected = pattern.match(left, collected)
-        return True, left, collected
-
-
-class OptionsShortcut(Optional):
-
-    """Marker/placeholder for [options] shortcut."""
-
-
-class OneOrMore(BranchPattern):
-
-    def match(self, left, collected=None):
-        assert len(self.children) == 1
-        collected = [] if collected is None else collected
-        l = left
-        c = collected
-        l_ = None
-        matched = True
-        times = 0
-        while matched:
-            # could it be that something didn't match but changed l or c?
-            matched, l, c = self.children[0].match(l, c)
-            times += 1 if matched else 0
-            if l_ == l:
-                break
-            l_ = l
-        if times >= 1:
-            return True, l, c
-        return False, left, collected
-
-
-class Either(BranchPattern):
-
-    def match(self, left, collected=None):
-        collected = [] if collected is None else collected
-        outcomes = []
-        for pattern in self.children:
-            matched, _, _ = outcome = pattern.match(left, collected)
-            if matched:
-                outcomes.append(outcome)
-        if outcomes:
-            return min(outcomes, key=lambda outcome: len(outcome[1]))
-        return False, left, collected
-
-
-class Tokens(list):
-
-    def __init__(self, source, error=DocoptExit):
-        self += source.split() if hasattr(source, 'split') else source
-        self.error = error
-
-    @staticmethod
-    def from_pattern(source):
-        source = re.sub(r'([\[\]\(\)\|]|\.\.\.)', r' \1 ', source)
-        source = [s for s in re.split('\s+|(\S*<.*?>)', source) if s]
-        return Tokens(source, error=DocoptLanguageError)
-
-    def move(self):
-        return self.pop(0) if len(self) else None
-
-    def current(self):
-        return self[0] if len(self) else None
-
-
-def parse_long(tokens, options):
-    """long ::= '--' chars [ ( ' ' | '=' ) chars ] ;"""
-    long, eq, value = tokens.move().partition('=')
-    assert long.startswith('--')
-    value = None if eq == value == '' else value
-    similar = [o for o in options if o.long == long]
-    if tokens.error is DocoptExit and similar == []:  # if no exact match
-        similar = [o for o in options if o.long and o.long.startswith(long)]
-    if len(similar) > 1:  # might be simply specified ambiguously 2+ times?
-        raise tokens.error('%s is not a unique prefix: %s?' %
-                           (long, ', '.join(o.long for o in similar)))
-    elif len(similar) < 1:
-        argcount = 1 if eq == '=' else 0
-        o = Option(None, long, argcount)
-        options.append(o)
-        if tokens.error is DocoptExit:
-            o = Option(None, long, argcount, value if argcount else True)
-    else:
-        o = Option(similar[0].short, similar[0].long,
-                   similar[0].argcount, similar[0].value)
-        if o.argcount == 0:
-            if value is not None:
-                raise tokens.error('%s must not have an argument' % o.long)
-        else:
-            if value is None:
-                if tokens.current() in [None, '--']:
-                    raise tokens.error('%s requires argument' % o.long)
-                value = tokens.move()
-        if tokens.error is DocoptExit:
-            o.value = value if value is not None else True
-    return [o]
-
-
-def parse_shorts(tokens, options):
-    """shorts ::= '-' ( chars )* [ [ ' ' ] chars ] ;"""
-    token = tokens.move()
-    assert token.startswith('-') and not token.startswith('--')
-    left = token.lstrip('-')
-    parsed = []
-    while left != '':
-        short, left = '-' + left[0], left[1:]
-        similar = [o for o in options if o.short == short]
-        if len(similar) > 1:
-            raise tokens.error('%s is specified ambiguously %d times' %
-                               (short, len(similar)))
-        elif len(similar) < 1:
-            o = Option(short, None, 0)
-            options.append(o)
-            if tokens.error is DocoptExit:
-                o = Option(short, None, 0, True)
-        else:  # why copying is necessary here?
-            o = Option(short, similar[0].long,
-                       similar[0].argcount, similar[0].value)
-            value = None
-            if o.argcount != 0:
-                if left == '':
-                    if tokens.current() in [None, '--']:
-                        raise tokens.error('%s requires argument' % short)
-                    value = tokens.move()
-                else:
-                    value = left
-                    left = ''
-            if tokens.error is DocoptExit:
-                o.value = value if value is not None else True
-        parsed.append(o)
-    return parsed
-
-
-def parse_pattern(source, options):
-    tokens = Tokens.from_pattern(source)
-    result = parse_expr(tokens, options)
-    if tokens.current() is not None:
-        raise tokens.error('unexpected ending: %r' % ' '.join(tokens))
-    return Required(*result)
-
-
-def parse_expr(tokens, options):
-    """expr ::= seq ( '|' seq )* ;"""
-    seq = parse_seq(tokens, options)
-    if tokens.current() != '|':
-        return seq
-    result = [Required(*seq)] if len(seq) > 1 else seq
-    while tokens.current() == '|':
-        tokens.move()
-        seq = parse_seq(tokens, options)
-        result += [Required(*seq)] if len(seq) > 1 else seq
-    return [Either(*result)] if len(result) > 1 else result
-
-
-def parse_seq(tokens, options):
-    """seq ::= ( atom [ '...' ] )* ;"""
-    result = []
-    while tokens.current() not in [None, ']', ')', '|']:
-        atom = parse_atom(tokens, options)
-        if tokens.current() == '...':
-            atom = [OneOrMore(*atom)]
-            tokens.move()
-        result += atom
-    return result
-
-
-def parse_atom(tokens, options):
-    """atom ::= '(' expr ')' | '[' expr ']' | 'options'
-             | long | shorts | argument | command ;
-    """
-    token = tokens.current()
-    result = []
-    if token in '([':
-        tokens.move()
-        matching, pattern = {'(': [')', Required], '[': [']', Optional]}[token]
-        result = pattern(*parse_expr(tokens, options))
-        if tokens.move() != matching:
-            raise tokens.error("unmatched '%s'" % token)
-        return [result]
-    elif token == 'options':
-        tokens.move()
-        return [OptionsShortcut()]
-    elif token.startswith('--') and token != '--':
-        return parse_long(tokens, options)
-    elif token.startswith('-') and token not in ('-', '--'):
-        return parse_shorts(tokens, options)
-    elif token.startswith('<') and token.endswith('>') or token.isupper():
-        return [Argument(tokens.move())]
-    else:
-        return [Command(tokens.move())]
-
-
-def parse_argv(tokens, options, options_first=False):
-    """Parse command-line argument vector.
-
-    If options_first:
-        argv ::= [ long | shorts ]* [ argument ]* [ '--' [ argument ]* ] ;
-    else:
-        argv ::= [ long | shorts | argument ]* [ '--' [ argument ]* ] ;
-
-    """
-    parsed = []
-    while tokens.current() is not None:
-        if tokens.current() == '--':
-            return parsed + [Argument(None, v) for v in tokens]
-        elif tokens.current().startswith('--'):
-            parsed += parse_long(tokens, options)
-        elif tokens.current().startswith('-') and tokens.current() != '-':
-            parsed += parse_shorts(tokens, options)
-        elif options_first:
-            return parsed + [Argument(None, v) for v in tokens]
-        else:
-            parsed.append(Argument(None, tokens.move()))
-    return parsed
-
-
-def parse_defaults(doc):
-    defaults = []
-    for s in parse_section('options:', doc):
-        # FIXME corner case "bla: options: --foo"
-        _, _, s = s.partition(':')  # get rid of "options:"
-        split = re.split('\n[ \t]*(-\S+?)', '\n' + s)[1:]
-        split = [s1 + s2 for s1, s2 in zip(split[::2], split[1::2])]
-        options = [Option.parse(s) for s in split if s.startswith('-')]
-        defaults += options
-    return defaults
-
-
-def parse_section(name, source):
-    pattern = re.compile('^([^\n]*' + name + '[^\n]*\n?(?:[ \t].*?(?:\n|$))*)',
-                         re.IGNORECASE | re.MULTILINE)
-    return [s.strip() for s in pattern.findall(source)]
-
-
-def formal_usage(section):
-    _, _, section = section.partition(':')  # drop "usage:"
-    pu = section.split()
-    return '( ' + ' '.join(') | (' if s == pu[0] else s for s in pu[1:]) + ' )'
-
-
-def extras(help, version, options, doc):
-    if help and any((o.name in ('-h', '--help')) and o.value for o in options):
-        print(doc.strip("\n"))
-        sys.exit()
-    if version and any(o.name == '--version' and o.value for o in options):
-        print(version)
-        sys.exit()
-
-
-class Dict(dict):
-    def __repr__(self):
-        return '{%s}' % ',\n '.join('%r: %r' % i for i in sorted(self.items()))
-
-
-def docopt(doc, argv=None, help=True, version=None, options_first=False):
-    """Parse `argv` based on command-line interface described in `doc`.
-
-    `docopt` creates your command-line interface based on its
-    description that you pass as `doc`. Such description can contain
-    --options, <positional-argument>, commands, which could be
-    [optional], (required), (mutually | exclusive) or repeated...
-
-    Parameters
-    ----------
-    doc : str
-        Description of your command-line interface.
-    argv : list of str, optional
-        Argument vector to be parsed. sys.argv[1:] is used if not
-        provided.
-    help : bool (default: True)
-        Set to False to disable automatic help on -h or --help
-        options.
-    version : any object
-        If passed, the object will be printed if --version is in
-        `argv`.
-    options_first : bool (default: False)
-        Set to True to require options precede positional arguments,
-        i.e. to forbid options and positional arguments intermix.
-
-    Returns
-    -------
-    args : dict
-        A dictionary, where keys are names of command-line elements
-        such as e.g. "--verbose" and "<path>", and values are the
-        parsed values of those elements.
-
-    Example
-    -------
-    >>> from docopt import docopt
-    >>> doc = '''
-    ... Usage:
-    ...     my_program tcp <host> <port> [--timeout=<seconds>]
-    ...     my_program serial <port> [--baud=<n>] [--timeout=<seconds>]
-    ...     my_program (-h | --help | --version)
-    ...
-    ... Options:
-    ...     -h, --help  Show this screen and exit.
-    ...     --baud=<n>  Baudrate [default: 9600]
-    ... '''
-    >>> argv = ['tcp', '127.0.0.1', '80', '--timeout', '30']
-    >>> docopt(doc, argv)
-    {'--baud': '9600',
-     '--help': False,
-     '--timeout': '30',
-     '--version': False,
-     '<host>': '127.0.0.1',
-     '<port>': '80',
-     'serial': False,
-     'tcp': True}
-
-    See also
-    --------
-    * For video introduction see http://docopt.org
-    * Full documentation is available in README.rst as well as online
-      at https://github.com/docopt/docopt#readme
-
-    """
-    argv = sys.argv[1:] if argv is None else argv
-
-    usage_sections = parse_section('usage:', doc)
-    if len(usage_sections) == 0:
-        raise DocoptLanguageError('"usage:" (case-insensitive) not found.')
-    if len(usage_sections) > 1:
-        raise DocoptLanguageError('More than one "usage:" (case-insensitive).')
-    DocoptExit.usage = usage_sections[0]
-
-    options = parse_defaults(doc)
-    pattern = parse_pattern(formal_usage(DocoptExit.usage), options)
-    # [default] syntax for argument is disabled
-    #for a in pattern.flat(Argument):
-    #    same_name = [d for d in arguments if d.name == a.name]
-    #    if same_name:
-    #        a.value = same_name[0].value
-    argv = parse_argv(Tokens(argv), list(options), options_first)
-    pattern_options = set(pattern.flat(Option))
-    for options_shortcut in pattern.flat(OptionsShortcut):
-        doc_options = parse_defaults(doc)
-        options_shortcut.children = list(set(doc_options) - pattern_options)
-        #if any_options:
-        #    options_shortcut.children += [Option(o.short, o.long, o.argcount)
-        #                    for o in argv if type(o) is Option]
-    extras(help, version, argv, doc)
-    matched, left, collected = pattern.fix().match(argv)
-    if matched and left == []:  # better error message if left?
-        return Dict((a.name, a.value) for a in (pattern.flat() + collected))
-    raise DocoptExit()
@@ -1,132 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""Usage:
-  pip-diff (--fresh | --stale) <reqfile1> <reqfile2> [--exclude <package>...]
-  pip-diff (-h | --help)
-
-Options:
-  -h --help     Show this screen.
-  --fresh       List newly added packages.
-  --stale       List removed packages.
-"""
-import os
-from docopt import docopt
-
-try:  # pip >= 10
-    from pip._internal.req import parse_requirements
-    from pip._internal.download import PipSession as session
-
-    def PackageFinder(find_links, index_urls, session=None):
-        from pip._internal.index import PackageFinder
-        from pip._internal.models.search_scope import SearchScope
-        from pip._internal.models.selection_prefs import SelectionPreferences
-
-        search_scope = SearchScope.create(find_links, index_urls)
-        selection_prefs = SelectionPreferences(allow_yanked=False)
-        return PackageFinder.create(search_scope, selection_prefs, session=session)
-
-except ImportError:  # pip <= 9.0.3
-    from pip.req import parse_requirements
-    from pip.index import PackageFinder
-    from pip._vendor.requests import session
-
-requests = session()
-
-
-class Requirements(object):
-    def __init__(self, reqfile=None):
-        super(Requirements, self).__init__()
-        self.path = reqfile
-        self.requirements = []
-
-        if reqfile:
-            self.load(reqfile)
-
-    def __repr__(self):
-        return '<Requirements \'{}\'>'.format(self.path)
-
-    def load(self, reqfile):
-        if not os.path.exists(reqfile):
-            raise ValueError('The given requirements file does not exist.')
-
-        finder = PackageFinder([], [], session=requests)
-        for requirement in parse_requirements(reqfile, finder=finder, session=requests):
-            if requirement.req:
-                if not getattr(requirement.req, 'name', None):
-                    # Prior to pip 8.1.2 the attribute `name` did not exist.
-                    requirement.req.name = requirement.req.project_name
-                    requirement.req.name = requirement.req.name.lower()
-                self.requirements.append(requirement.req)
-
-
-    def diff(self, requirements, ignore_versions=False, excludes=None):
-        r1 = self
-        r2 = requirements
-        results = {'fresh': [], 'stale': []}
-
-        # Generate fresh packages.
-        other_reqs = (
-            [r.name for r in r1.requirements]
-            if ignore_versions else r1.requirements
-        )
-
-        for req in r2.requirements:
-            r = req.name if ignore_versions else req
-
-            if r not in other_reqs and r not in excludes:
-                results['fresh'].append(req)
-
-        # Generate stale packages.
-        other_reqs = (
-            [r.name for r in r2.requirements]
-            if ignore_versions else r2.requirements
-        )
-
-        for req in r1.requirements:
-            r = req.name if ignore_versions else req
-
-            if r not in other_reqs and r not in excludes:
-                results['stale'].append(req)
-
-        return results
-
-
-def diff(r1, r2, include_fresh=False, include_stale=False, excludes=None):
-    include_versions = True if include_stale else False
-    excludes = excludes if len(excludes) else []
-
-    try:
-        r1 = Requirements(r1)
-        r2 = Requirements(r2)
-    except ValueError:
-        print('There was a problem loading the given requirements files.')
-        exit(os.EX_NOINPUT)
-
-    results = r1.diff(r2, ignore_versions=True, excludes=excludes)
-
-    if include_fresh:
-        for line in results['fresh']:
-            print(line.name if include_versions else line)
-
-    if include_stale:
-        for line in results['stale']:
-            print(line.name if include_versions else line)
-
-
-def main():
-    args = docopt(__doc__, version='pip-diff')
-
-    kwargs = {
-        'r1': args['<reqfile1>'],
-        'r2': args['<reqfile2>'],
-        'include_fresh': args['--fresh'],
-        'include_stale': args['--stale'],
-        'excludes': args['<package>']
-    }
-
-    diff(**kwargs)
-
-
-if __name__ == '__main__':
-    main()
@@ -1,94 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""Usage:
-  pip-grep [-s] <reqfile> <package>...
-
-Options:
-  -h --help     Show this screen.
-"""
-import os
-import sys
-from docopt import docopt
-
-try:  # pip >= 10
-    from pip._internal.req import parse_requirements
-    from pip._internal.download import PipSession as session
-
-    def PackageFinder(find_links, index_urls, session=None):
-        from pip._internal.index import PackageFinder
-        from pip._internal.models.search_scope import SearchScope
-        from pip._internal.models.selection_prefs import SelectionPreferences
-
-        search_scope = SearchScope.create(find_links, index_urls)
-        selection_prefs = SelectionPreferences(allow_yanked=False)
-        return PackageFinder.create(search_scope, selection_prefs, session=session)
-
-except ImportError:  # pip <= 9.0.3
-    from pip.req import parse_requirements
-    from pip.index import PackageFinder
-    from pip._vendor.requests import session
-
-
-requests = session()
-
-
-class Requirements(object):
-    def __init__(self, reqfile=None):
-        super(Requirements, self).__init__()
-        self.path = reqfile
-        self.requirements = []
-
-        if reqfile:
-            self.load(reqfile)
-
-    def __repr__(self):
-        return '<Requirements \'{}\'>'.format(self.path)
-
-    def load(self, reqfile):
-        if not os.path.exists(reqfile):
-            raise ValueError('The given requirements file does not exist.')
-
-        finder = PackageFinder([], [], session=requests)
-        for requirement in parse_requirements(reqfile, finder=finder, session=requests):
-            if requirement.req:
-                if not getattr(requirement.req, 'name', None):
-                    # Prior to pip 8.1.2 the attribute `name` did not exist.
-                    requirement.req.name = requirement.req.project_name
-                self.requirements.append(requirement.req)
-
-
-def grep(reqfile, packages, silent=False):
-    try:
-        r = Requirements(reqfile)
-    except ValueError:
-        if not silent:
-            print('There was a problem loading the given requirement file.')
-        exit(os.EX_NOINPUT)
-
-    for req in r.requirements:
-        if req.name in packages:
-            if not silent:
-                print('Package {} found!'.format(req.name))
-            exit(0)
-
-    if not silent:
-        print('Not found.')
-
-    exit(1)
-
-
-def main():
-    args = docopt(__doc__, version='pip-grep')
-
-    kwargs = {'reqfile': args['<reqfile>'], 'packages': args['<package>'], 'silent': args['-s']}
-
-    grep(**kwargs)
-
-
-if __name__ == '__main__':
-    try:
-        main()
-    except Exception:
-        sys.exit(1)
-
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""Usage:
-  sp-grep [-s] <package>...
-
-Options:
-  -h --help     Show this screen.
-"""
-from docopt import docopt
-from pkg_resources import DistributionNotFound, get_distribution
-
-
-def has_any_distribution(names, silent=False):
-    for name in names:
-        try:
-            get_distribution(name)
-        except DistributionNotFound:
-            continue
-
-        if not silent:
-            print('Package {name} found!'.format(name=name))
-
-        exit(0)
-
-    if not silent:
-        print('Not found.')
-
-    exit(1)
-
-
-def main():
-    args = docopt(__doc__, version='sp-grep')
-    has_any_distribution(names=args['<package>'], silent=args['-s'])
-
-
-if __name__ == '__main__':
-    main()
				`@@ -0,0 +1 @@`
				`hg+https://www.mercurial-scm.org/repo/python-hglib/#egg=python-hglib`