Release v182 (#1091)

https://pythoninsider.blogspot.com/2020/10/python-390-is-now-available-and-you-can.html
https://www.python.org/downloads/release/python-390/
https://docs.python.org/release/3.9.0/whatsnew/3.9.html

Binaries generated using:

```
make deploy-runtimes RUNTIMES='python-3.9.0' STACKS='heroku-16 heroku-18' ENV_FILE=...
```

Closes @W-7791272@.

.travis.yml

@@ -1,5 +1,5 @@
 language: minimal
-dist: bionic
+dist: focal
 branches:
   only:
   - main

CHANGELOG.md

@@ -3,6 +3,36 @@
 ## Unreleased
 
 
+## v182 (2020-10-06)
+
+- Python 3.9.0 is now available (CPython) (#1090).
+- Migrate from the `lang-python` S3 bucket to `heroku-buildpack-python` (#1089).
+- Remove `vendor/shunit2` (#1086).
+- Replace `BUILDPACK_VENDOR_URL` and `USE_STAGING_BINARIES` with `BUILDPACK_S3_BASE_URL` (#1085).
+
+## v181 (2020-09-29)
+
+- PyPy 2.7 and 3.6, version 7.3.2 are now available (Note: PyPy support is in beta) (#1081).
+
+## v180 (2020-09-24)
+
+- Python 3.8.6 is now available (CPython) (#1072).
+
+## v179 (2020-09-23)
+
+- Remove duplicate pipenv metric event (#1070).
+- Emit metrics for how the Python version was chosen for an app (#1069).
+- Emit Python version metric events for all builds, not just clean installs (#1066).
+
+## v178 (2020-09-07)
+
+- Python 3.5.10 is now available (CPython) (#1062).
+
+## v177 (2020-08-18)
+
+- Python 3.6.12 and 3.7.9 are now available (CPython) (#1054).
+- The default Python version for new apps is now 3.6.12 (previously 3.6.11) (#1054).
+
 ## v176 (2020-08-12)
 
 - Rebuild the Python 3.4.10 archives with the correct version of Python (#1048).

Gemfile.lock

@@ -9,7 +9,7 @@ GEM
       excon
       moneta
       multi_json (>= 1.9.2)
-    heroku_hatchet (7.0.0)
+    heroku_hatchet (7.3.0)
       excon (~> 0)
       platform-api (~> 3)
       rrrretry (~> 1)

Makefile

@@ -1,15 +1,18 @@
 # These targets are not files
-.PHONY: check test buildenv-heroku-16 buildenv-heroku-18 tools
+.PHONY: check test builder-image buildenv deploy-runtimes tools
 
 STACK ?= heroku-18
+STACKS ?= cedar-14 heroku-16 heroku-18
 TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
+ENV_FILE ?= builds/dockerenv.default
+BUILDER_IMAGE_PREFIX := heroku-python-build
 
 ifeq ($(STACK),cedar-14)
 	# Cedar-14 doesn't have a build image varient.
-	IMAGE_TAG := heroku/cedar:14
+	STACK_IMAGE_TAG := heroku/cedar:14
 else
 	# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
-	IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
+	STACK_IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
 endif
 
 check:
@@ -19,37 +22,42 @@ check:
 
 test:
 	@echo "Running tests using: STACK=$(STACK) TEST_CMD='$(TEST_CMD)'"
-	@echo ""
-	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
-	@echo ""
-
-buildenv-heroku-16:
-	@echo "Creating build environment (heroku-16)..."
 	@echo
-	@docker build --pull -f $(shell pwd)/builds/heroku-16.Dockerfile -t python-buildenv-heroku-16 .
+	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(STACK_IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
+	@echo
+
+builder-image:
+	@echo "Generating binary builder image for $(STACK)..."
+	@echo
+	@docker build --pull -f builds/$(STACK).Dockerfile -t "$(BUILDER_IMAGE_PREFIX)-$(STACK)" .
+	@echo
+
+buildenv: builder-image
+	@echo "Starting build environment for $(STACK)..."
 	@echo
 	@echo "Usage..."
 	@echo
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ bob build runtimes/python-2.7.13"
-	@echo "  $$ bob deploy runtimes/python-2.7.13"
+	@echo "  $$ bob build runtimes/python-X.Y.Z"
 	@echo
-	@docker run -it --rm python-buildenv-heroku-16
+	@docker run --rm -it --env-file="$(ENV_FILE)" -v $(PWD)/builds:/app/builds "$(BUILDER_IMAGE_PREFIX)-$(STACK)" bash
 
-buildenv-heroku-18:
-	@echo "Creating build environment (heroku-18)..."
+deploy-runtimes:
+ifndef RUNTIMES
+	$(error No runtimes specified! Use: "make deploy-runtimes RUNTIMES='python-X.Y.Z ...' [STACKS='heroku-18 ...'] [ENV_FILE=...]")
+endif
+	@echo "Using: RUNTIMES='$(RUNTIMES)' STACKS='$(STACKS)' ENV_FILE='$(ENV_FILE)'"
 	@echo
-	@docker build --pull -f $(shell pwd)/builds/heroku-18.Dockerfile -t python-buildenv-heroku-18 .
-	@echo
-	@echo "Usage..."
-	@echo
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ bob build runtimes/python-2.7.13"
-	@echo "  $$ bob deploy runtimes/python-2.7.13"
-	@echo
-	@docker run -it --rm python-buildenv-heroku-18
+	@set -eu; for stack in $(STACKS); do \
+		$(MAKE) builder-image STACK=$${stack}; \
+		for runtime in $(RUNTIMES); do \
+			echo "Generating/deploying $${runtime} for $${stack}..."; \
+			echo; \
+			docker run --rm -it --env-file="$(ENV_FILE)" "$(BUILDER_IMAGE_PREFIX)-$${stack}" bob deploy "runtimes/$${runtime}"; \
+			echo; \
+		done; \
+	done
 
 tools:
 	git clone https://github.com/kennethreitz/pip-pop.git
 	mv pip-pop/bin/* vendor/pip-pop/
-	rm -fr pip-pop
+	rm -rf pip-pop

README.md

@@ -26,7 +26,7 @@ remote: Compressing source files... done.
 remote: Building source:
 remote:
 remote: -----> Python app detected
-remote: -----> Installing python-3.7.4
+remote: -----> Installing python
 remote: -----> Installing pip
 remote: -----> Installing SQLite3
 remote: -----> Installing requirements with pip
@@ -44,7 +44,7 @@ A `requirements.txt` must be present at the root of your application's repositor
 
 To specify your python version, you also need a `runtime.txt` file - unless you are using the default Python runtime version.
 
-Current default Python Runtime: Python 3.6.9
+Current default Python Runtime: Python 3.6.12
 
 Alternatively, you can provide a `setup.py` file, or a `Pipfile`.
 Using `pipenv` will generate `runtime.txt` at build time if one of the field `python_version` or `python_full_version` is specified in the `requires` section of your `Pipfile`.
@@ -62,9 +62,10 @@ Specify a Python Runtime
 
 Supported runtime options include:
 
-- `python-3.8.5`
-- `python-3.7.8`
-- `python-3.6.11`
+- `python-3.9.0`
+- `python-3.8.6`
+- `python-3.7.9`
+- `python-3.6.12`
 - `python-2.7.18`
 
 ## Tests

bin/compile

@@ -37,23 +37,27 @@ ENV_DIR=$3
 # Export Path variables, for use in sub-scripts.
 export BUILD_DIR CACHE_DIR ENV_DIR
 
-# Set the Buildpack's internet target for downloading Python distributions.
-# The user can provide BUILDPACK_VENDOR_URL to specify a custom target.
+# Set the base URL for downloading buildpack assets like Python runtimes.
+# The user can provide BUILDPACK_S3_BASE_URL to specify a custom target.
 # Note: this is designed for non-Heroku use, as it does not use the user-provided
 # environment variable mechanism (the ENV_DIR).
-VENDOR_URL="https://lang-python.s3.amazonaws.com/$STACK"
-if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then
-    VENDOR_URL="$BUILDPACK_VENDOR_URL"
-elif [[ -n ${USE_STAGING_BINARIES} ]]; then
-    VENDOR_URL="$USE_STAGING_BINARIES/$STACK"
+if [[ "${STACK}" == "cedar-14" ]]; then
+  # Since Cedar-14 is EOL on 2020-11-02 it's not worth migrating it to the new bucket.
+  DEFAULT_S3_BASE_URL='https://lang-python.s3.amazonaws.com'
+else
+  DEFAULT_S3_BASE_URL='https://heroku-buildpack-python.s3.amazonaws.com'
 fi
-export VENDOR_URL
+S3_BASE_URL="${BUILDPACK_S3_BASE_URL:-${DEFAULT_S3_BASE_URL}}"
+# This has to be exported since it's used by the geo-libs step which is run in a subshell.
+# TODO: Stop exporting once the geo-libs step is removed or no longer uses `sub_env`.
+export S3_BASE_URL
 
 # Default Python Versions
 # shellcheck source=bin/default_pythons
 source "$BIN_DIR/default_pythons"
 
 # Supported Python Branches
+PY39="python-3.9"
 PY38="python-3.8"
 PY37="python-3.7"
 PY36="python-3.6"
@@ -189,7 +193,7 @@ source "$BIN_DIR/steps/hooks/pre_compile"
 # continue to use that version of Python in perpituity (warnings will be raised if
 # they are out–of–date).
 if [ -f "$CACHE_DIR/.heroku/python-version" ]; then
-  DEFAULT_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
+  CACHED_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
 fi
 
 # We didn't always record the stack version. This code is in place because of that.
@@ -206,9 +210,14 @@ fi
 # shellcheck source=bin/steps/pipenv-python-version
 source "$BIN_DIR/steps/pipenv-python-version"
 
-# If no runtime was provided by the user, assume the default Python runtime version.
-if [ ! -f runtime.txt ]; then
-  echo "$DEFAULT_PYTHON_VERSION" > runtime.txt
+if [[ -f runtime.txt ]]; then
+  mcount "version.reason.python.specified"
+elif [[ -n "${CACHED_PYTHON_VERSION:-}" ]]; then
+  mcount "version.reason.python.cached"
+  echo "${CACHED_PYTHON_VERSION}" > runtime.txt
+else
+  mcount "version.reason.python.default"
+  echo "${DEFAULT_PYTHON_VERSION}" > runtime.txt
 fi
 
 # Create the directory for .profile.d, if it doesn't exist.
@@ -304,7 +313,7 @@ mtime "nltk.download.time" "${start}"
 # and copying it into the proper place (the logical place to do this was early, but it must be done here).
 # In CI, $BUILD_DIR is /app.
 if [[ ! "$BUILD_DIR" == "/app" ]]; then
-  rm -fr "$BUILD_DIR/.heroku/src"
+  rm -rf "$BUILD_DIR/.heroku/src"
   deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src"
 fi
 

bin/default_pythons

@@ -5,12 +5,13 @@
 # the env vars to subprocesses.
 # shellcheck disable=2034
 
-DEFAULT_PYTHON_VERSION="python-3.6.11"
-LATEST_38="python-3.8.5"
-LATEST_37="python-3.7.8"
-LATEST_36="python-3.6.11"
-LATEST_35="python-3.5.9"
+DEFAULT_PYTHON_VERSION="python-3.6.12"
+LATEST_39="python-3.9.0"
+LATEST_38="python-3.8.6"
+LATEST_37="python-3.7.9"
+LATEST_36="python-3.6.12"
+LATEST_35="python-3.5.10"
 LATEST_34="python-3.4.10"
 LATEST_27="python-2.7.18"
-LATEST_PYPY_36="pypy3.6-7.3.1"
-LATEST_PYPY_27="pypy2.7-7.3.1"
+LATEST_PYPY_36="pypy3.6-7.3.2"
+LATEST_PYPY_27="pypy2.7-7.3.2"

bin/steps/gdal

@@ -10,7 +10,7 @@
 # This script is invoked by [`bin/compile`](/).
 
 # The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
+VENDORED_GDAL="${S3_BASE_URL}/${STACK}/libraries/vendor/gdal.tar.gz"
 
 PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 

bin/steps/geo-libs

@@ -10,9 +10,9 @@
 # This script is invoked by [`bin/compile`](/).
 
 # The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
-VENDORED_GEOS="${VENDOR_URL}/libraries/vendor/geos.tar.gz"
-VENDORED_PROJ="${VENDOR_URL}/libraries/vendor/proj.tar.gz"
+VENDORED_GDAL="${S3_BASE_URL}/${STACK}/libraries/vendor/gdal.tar.gz"
+VENDORED_GEOS="${S3_BASE_URL}/${STACK}/libraries/vendor/geos.tar.gz"
+VENDORED_PROJ="${S3_BASE_URL}/${STACK}/libraries/vendor/proj.tar.gz"
 
 PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 

bin/steps/pip-uninstall

@@ -16,7 +16,7 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
       mcount "failure.bad-requirements"
     fi
 
-    rm -fr requirements-declared.txt
+    rm -rf requirements-declared.txt
 
     if [[ -s .heroku/python/requirements-stale.txt ]]; then
       puts-step "Uninstalling stale dependencies"

bin/steps/pipenv

@@ -9,10 +9,7 @@ set -e
 if [[ -f Pipfile.lock ]]; then
     if [[ -f .heroku/python/Pipfile.lock.sha256 ]]; then
         if [[ $(openssl dgst -sha256 Pipfile.lock) == $(cat .heroku/python/Pipfile.lock.sha256) ]]; then
-            # Measure that we're using Pipenv.
-            mcount "tool.pipenv"
-
-            # Don't skip installation of there are git deps.
+            # Don't skip installation if there are git deps.
             if ! grep -q 'git' Pipfile.lock; then
                 echo "Skipping installation, as Pipfile.lock hasn't changed since last deploy." | indent
 

bin/steps/pipenv-python-version

@@ -30,6 +30,9 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
                 if [ "$PYTHON" = 3.8 ]; then
                     echo "$LATEST_38" > "$BUILD_DIR/runtime.txt"
                 fi
+                if [ "$PYTHON" = 3.9 ]; then
+                    echo "$LATEST_39" > "$BUILD_DIR/runtime.txt"
+                fi
             fi
 
 

bin/steps/pylibmc

@@ -15,7 +15,7 @@ if [[ "$STACK" != "cedar-14" ]]; then
 fi
 
 # The location of the pre-compiled libmemcached binary.
-VENDORED_MEMCACHED="${VENDOR_URL}/libraries/vendor/libmemcache.tar.gz"
+VENDORED_MEMCACHED="${S3_BASE_URL}/${STACK}/libraries/vendor/libmemcache.tar.gz"
 
 # Syntax sugar.
 # shellcheck source=bin/utils

bin/steps/python

@@ -5,7 +5,7 @@ runtime-fixer runtime.txt
 PYTHON_VERSION=$(cat runtime.txt)
 
 # The location of the pre-compiled python binary.
-VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"
+VENDORED_PYTHON="${S3_BASE_URL}/${STACK}/runtimes/${PYTHON_VERSION}.tar.gz"
 
 SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
 SECURITY_UPDATE_PYPY="The PyPy project has released a security update! Please consider upgrading to"
@@ -16,6 +16,12 @@ PYTHON_2_EOL_UPDATE="Python 2 has reached it's community EOL. Upgrade your Pytho
 
 # check if runtime exists
 if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
+  if [[ "$PYTHON_VERSION" == $PY39* ]]; then
+    if [ "$PYTHON_VERSION" != "$LATEST_39" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_39"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
   if [[ "$PYTHON_VERSION" == $PY38* ]]; then
     # do things to alert the user of security release available
     if [ "$PYTHON_VERSION" != "$LATEST_38" ]; then
@@ -52,11 +58,9 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
     fi
   fi
   if [[ "$PYTHON_VERSION" == $PY27* ]]; then
+    puts-warn "$PYTHON_2_EOL_UPDATE"
+    echo "       Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq"
     # security update note
-    if [[ "$(date "+%Y")" -gt "2019" ]]; then
-      puts-warn "$PYTHON_2_EOL_UPDATE"
-      echo "       Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq"
-    fi
     if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then
       puts-warn "$ONLY_SUPPORTED_2_VERSION" "$LATEST_27"
       echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
@@ -82,15 +86,17 @@ else
   exit 1
 fi
 
+mcount "version.python.${PYTHON_VERSION}"
+
 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
     puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache"
-    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
+    rm -rf .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
 fi
 
 if [ -f .heroku/python-version ]; then
   if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then
       puts-step "Found $(cat .heroku/python-version), removing"
-      rm -fr .heroku/python
+      rm -rf .heroku/python
   else
     SKIP_INSTALL=1
   fi
@@ -121,8 +127,6 @@ if [ ! "$SKIP_INSTALL" ]; then
     # Prepare destination directory.
     mkdir -p .heroku/python
 
-    mcount "version.python.$PYTHON_VERSION"
-
     if ! curl "${VENDORED_PYTHON}" -s | tar zxv -C .heroku/python &> /dev/null; then
       puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)."
       puts-warn "Aborting.  More info: https://devcenter.heroku.com/articles/python-support"
@@ -162,7 +166,7 @@ fi
 # Instead, we use the pip wheel to install itself, using the method described here:
 # https://github.com/pypa/pip/issues/2351#issuecomment-69994524
 PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl"
-PIP_WHEEL_URL="https://lang-python.s3.amazonaws.com/common/${PIP_WHEEL_FILENAME}"
+PIP_WHEEL_URL="${S3_BASE_URL}/common/${PIP_WHEEL_FILENAME}"
 PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}"
 
 if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then

builds/README.md

@@ -1,57 +1,80 @@
 # Python Buildpack Binaries
 
-## Building the Docker Images
-
-**After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack:
-
-    $ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
-    $ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-    $ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-
-## Using the Image
-
-You can e.g. `bash` into each of the images you built using their tag:
-
-    docker run --rm -ti heroku-python-build-cedar-14 bash
-    docker run --rm -ti heroku-python-build-heroku-16 bash
-    docker run --rm -ti heroku-python-build-heroku-18 bash
-
-You then have a shell where you can run `bob build`, `bob deploy`, and so forth. You can of course also invoke these programs directly with `docker run`:
-
-    docker run --rm -ti heroku-python-build-heroku-18 bob build runtimes/python-2.7.15
-
-In order to `bob deploy`, AWS credentials must be set up, as well as name and prefix of your custom S3 bucket (unless you're deploying to the Heroku production buckets that are pre-defined in each `Dockerfile`); see next section for details.
+The binaries for this buildpack are built in Docker containers based on the Heroku stack image.
 
 ## Configuration
 
-File `dockerenv.default` contains a list of required env vars; most of these have default values defined in `Dockerfile`. You can copy this file to a location outside the buildpack and modify it with the values you desire and pass its location with `--env-file`, or pass the env vars to `docker run` using `--env`.
+In order to publish binaries AWS credentials must be passed to the build container.
+If you are testing only the build (ie: `bob build`), these are optional.
 
-Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCKET`, `S3_PREFIX`, and `S3_REGION`. If you're building your own packages, you'll likely want to change `S3_BUCKET` and `S3_PREFIX` to match your info. Instead of setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` into that file, you may also pass them to `docker run` through the environment, or explicitly using `--env`, in order to prevent accidental commits of credentials.
+In addition, unless you are building the official binaries for Heroku (which use the defaults
+specified in each `Dockerfile`), you will need to override `S3_BUCKET` and `S3_PREFIX` to
+match your own S3 bucket/use case.
 
-### Passing AWS credentials to the container
+If you only need to set AWS credentials, you can do so by setting the environment variables
+`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` before calling the make commands.
 
-If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file.
+For example:
 
-#### Passing credentials explicitly
+```bash
+set +o history # Disable bash history
+export AWS_ACCESS_KEY_ID=...
+export AWS_SECRET_ACCESS_KEY=...
+set -o history # Re-enable bash history
+make ...
+```
 
-    docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash
+If you need to override the default S3 bucket, or would prefer not to use credentials via
+environment variables, then you need to instead use a Docker env file like so:
 
-#### Passing credentials through  the environment
+1. Copy the `builds/dockerenv.default` env file to a location outside the buildpack repository.
+2. Edit the new file, adding at a minimum the values for the variables
+   `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (see Docker
+   [env-file documentation](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file)).
+3. Pass the path of the file to the make commands using `ENV_FILE`. For example:
 
-The two environment variables `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY` are defined in `builds/dockerenv.default`, without values. This will cause Docker to "forward" values for these variables from the current environment, so you can pass them in:
+   ```bash
+   make ... ENV_FILE=~/.dockerenv.python-buildpack
+   ```
 
-    AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+## Launching an interactive build environment
 
-or
+To start an interactive version of the build environment (ideal for development) use the
+`buildenv` make target, passing in the desired `STACK` name. For example:
 
-    export AWS_ACCESS_KEY_ID=...
-    export AWS_SECRET_ACCESS_KEY=...
-    docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+```bash
+make buildenv STACK=heroku-18
+```
 
-#### Passing credentials through a separate env file
+This will create the builder docker image based on the latest image for that stack, and
+then start a bash shell where you can run `bob build`, `bob deploy`, and so forth.
 
-This method is the easiest for users who want to build packages in their own S3 bucket, as they will have to adjust the `S3_BUCKET` and `S3_PREFIX` environment variable values anyway from their default values.
+The `builds/` directory is bind-mounted into the running container, so local build formula
+changes will appear there immediately without the need to rebuild the image.
 
-For this method, it is important to keep the credentials file in a location outside the buildpack, so that your credentials aren't accidentally committed. Copy `builds/dockerenv.default` **to a safe location outside the buildpack directory**, and insert your values for `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY`.
+## Bulk deploying runtimes
 
-    docker run --rm -ti --env-file=../SOMEPATHOUTSIDE/s3.env heroku-python-build-heroku-18 bash
+When a new Python version is released, binaries have to be generated for multiple stacks.
+To automate this, use the `deploy-runtimes` make target, which will ensure the builder
+image is up to date, and then run `bob deploy` for each runtime-stack combination.
+
+The build formula name(s) are passed using `RUNTIMES`, like so:
+
+```bash
+make deploy-runtimes RUNTIMES='python-X.Y.Z'
+```
+
+By default this will deploy to all supported stacks (see `STACKS` in `Makefile`),
+but this can be overridden using `STACKS`:
+
+```bash
+make deploy-runtimes RUNTIMES='python-X.Y.Z' STACKS='heroku-16 heroku-18'
+```
+
+Multiple runtimes can also be specified (useful for when adding a new stack), like so:
+
+```bash
+make deploy-runtimes RUNTIMES='python-A.B.C python-X.Y.Z' STACKS='heroku-20'
+```
+
+Note: Both `RUNTIMES` and `STACKS` are space delimited.

builds/cedar-14.Dockerfile

@@ -1,15 +1,21 @@
 FROM heroku/cedar:14
 
-WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
     S3_BUCKET="lang-python" \
     S3_PREFIX="cedar-14/" \
-    DEBIAN_FRONTEND=noninteractive \
     STACK="cedar-14"
 
-RUN apt-get update && apt-get install -y python-pip libsqlite3-dev realpath && rm -rf /var/lib/apt/lists/*
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        realpath \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
 
 COPY requirements.txt /app/
-RUN pip install -r /app/requirements.txt
+# Can't use `--disable-pip-version-check --no-cache-dir` since not supported by Ubuntu 14.04's pip.
+RUN pip3 install -r /app/requirements.txt
 
 COPY . /app

builds/dockerenv.default

@@ -1,5 +1,8 @@
+# Since no values are specified here, these variables will be read from the environment at run time:
+# https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file
 AWS_ACCESS_KEY_ID
 AWS_SECRET_ACCESS_KEY
-S3_BUCKET
-S3_PREFIX
-S3_REGION
+
+# Uncomment these if you need to override the default S3 bucket and/or path prefixes.
+# S3_BUCKET
+# S3_PREFIX

builds/heroku-16.Dockerfile

@@ -1,15 +1,20 @@
 FROM heroku/heroku:16-build
 
-WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
-    S3_BUCKET="lang-python" \
+    S3_BUCKET="heroku-buildpack-python" \
     S3_PREFIX="heroku-16/" \
-    DEBIAN_FRONTEND=noninteractive \
     STACK="heroku-16"
 
-RUN apt-get update && apt-get install -y python-pip libsqlite3-dev && rm -rf /var/lib/apt/lists/*
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        python3-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
 
 COPY requirements.txt /app/
-RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
 
 COPY . /app

builds/heroku-18.Dockerfile

@@ -1,15 +1,20 @@
 FROM heroku/heroku:18-build
 
-WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
-    S3_BUCKET="lang-python" \
+    S3_BUCKET="heroku-buildpack-python" \
     S3_PREFIX="heroku-18/" \
-    DEBIAN_FRONTEND=noninteractive \
     STACK="heroku-18"
 
-RUN apt-get update && apt-get install --no-install-recommends -y python-pip-whl=9.0.1-2 python-pip=9.0.1-2 python-setuptools python-wheel libsqlite3-dev && rm -rf /var/lib/apt/lists/*
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        python3-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
 
 COPY requirements.txt /app/
-RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
 
 COPY . /app

builds/libraries/sqlite

@@ -19,4 +19,4 @@ make install
 
 # Cleanup
 cd ..
-rm -fr sqlite
+rm -rf sqlite

builds/runtimes/pypy2.7

@@ -13,11 +13,11 @@ dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full ver
 dep_package=${dep_name}-v${dep_version} # it's always "pypy2-…"
 dep_dirname=${dep_package}-linux64
 dep_archive_name=${dep_dirname}.tar.bz2
-dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+dep_url="https://downloads.python.org/pypy/${dep_archive_name}"
 
 echo "Building PyPy…"
 echo "${dep_url}"
 
-curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+curl -fL "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
 
 ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python"

builds/runtimes/pypy2.7-7.3.2

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy2.7

builds/runtimes/pypy3.6

@@ -13,11 +13,11 @@ dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full ver
 dep_package=${dep_name}${dep_version_prefix:-}-v${dep_version}${dep_version_suffix:-}
 dep_dirname=${dep_package}-linux64
 dep_archive_name=${dep_dirname}.tar.bz2
-dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+dep_url="https://downloads.python.org/pypy/${dep_archive_name}"
 
 echo "Building PyPy3…"
 echo "${dep_url}"
 
-curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+curl -fL "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
 
 ln "$OUT_PREFIX/bin/pypy3" "$OUT_PREFIX/bin/python"

builds/runtimes/pypy3.6-7.3.2

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy3.6

builds/runtimes/python-3.5.10

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.6.12

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.7.9

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.6

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.9.0

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

hatchet.lock

@@ -1,5 +1,5 @@
 ---
 - - "./repos/python/python-getting-started"
-  - master
+  - main
 - - "./repos/python/python_default"
   - ca947f69027b2a30be5d26f9a42f25e54f4d7a1a

requirements.txt

@@ -1,3 +1,6 @@
-docopt==0.6.2
-bob-builder==0.0.18
+# Dependencies for generating/publishing Python binaries.
+bob-builder==0.0.19
+
+# Sub-dependencies of bob-builder.
 boto==2.49.0
+docopt==0.6.2

test/fixtures/pypy2_7/runtime.txt

@@ -1 +1 @@
-pypy2.7-7.3.1
+pypy2.7-7.3.2

test/fixtures/pypy3_6/runtime.txt

@@ -1 +1 @@
-pypy3.6-7.3.1
+pypy3.6-7.3.2

test/fixtures/python2_fail/requirements.txt

@@ -1 +0,0 @@
-requests

test/fixtures/python2_fail/runtime.txt

@@ -1 +0,0 @@
-python-2.7.99

test/fixtures/python3_4_fail/requirements.txt

@@ -1 +0,0 @@
-flask

test/fixtures/python3_4_fail/runtime.txt

@@ -1 +0,0 @@
-python-3.4.99

test/fixtures/python3_5/runtime.txt

@@ -1 +1 @@
-python-3.5.9
+python-3.5.10

test/fixtures/python3_5_fail/requirements.txt

@@ -1 +0,0 @@
-flask

test/fixtures/python3_5_fail/runtime.txt

@@ -1 +0,0 @@
-python-3.5.99

test/fixtures/python3_6_fail/requirements.txt

@@ -1 +0,0 @@
-flask

test/fixtures/python3_6_fail/runtime.txt

@@ -1 +0,0 @@
-python-3.6.99

test/fixtures/python3_7_fail/requirements.txt

@@ -1 +0,0 @@
-flask

test/fixtures/python3_7_fail/runtime.txt

@@ -1 +0,0 @@
-python-3.7.99

test/fixtures/python3_8_fail/requirements.txt

@@ -1 +0,0 @@
-flask

test/fixtures/python3_9/runtime.txt

@@ -0,0 +1 @@
+python-3.9.0

test/fixtures/python3_9_warn/runtime.txt

@@ -0,0 +1 @@
+python-3.9.0

test/run-deps

@@ -43,7 +43,9 @@ testNLTK() {
   echo 'ignore::RuntimeWarning' > "${env_dir}/PYTHONWARNINGS"
   compile 'nltk' '' "${env_dir}"
   assertCaptured "[nltk_data] Downloading package city_database" "STD_ERR"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since the NLTK downloader outputs all
+  # progress/status messages to stderr (W-8146040).
+  assertCapturedSuccessWithStdErr
 }
 
 testPsycopg2() {

test/run-features

@@ -33,7 +33,9 @@ testStackChange() {
 testSetupPy() {
   compile "setup-py"
   assertCaptured "maya"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
 
 testStandardRequirements() {
@@ -45,29 +47,39 @@ testStandardRequirements() {
 testPipenv() {
   compile "pipenv"
   assertCaptured "Installing pip 9.0.2, setuptools 47.1.1 and wheel 0.34.2"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
 
 testPipenvLock() {
   compile "pipenv-lock"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
 
 testPipenvVersion() {
   compile "pipenv-version"
   assertCaptured $DEFAULT_PYTHON_VERSION
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
 
 testPipenvVersion2() {
   compile "pipenv-version2"
   assertCaptured $LATEST_27
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
 testPipenvFullVersion() {
   compile "pipenv-full-version"
   assertCaptured "3.6.3"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since stderr contains:
+  # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
+  assertCapturedSuccessWithStdErr
 }
 
 testNoRequirements() {
@@ -113,10 +125,10 @@ testHooks() {
     PROFILE_PATH
     PWD
     PYTHONUNBUFFERED
+    S3_BASE_URL
     SHLVL
     SOME_APP_CONFIG_VAR
     STACK
-    VENDOR_URL
   )
   if [[ "${STACK}" == "cedar-14" || "${STACK}" == "heroku-16" ]]; then
     # Remove "OLDPWD" from expected_env_vars since for bash <4.4 it's not exported to subshells:

test/run-versions

@@ -19,11 +19,7 @@ testPython2() {
     echo $LATEST_27 > "runtime.txt"
     compile "python2"
     assertCaptured $LATEST_27
-    if [[ $(date "+%Y") > "2019" ]]; then
-      assertCaptured "python-2-7-eol-faq";
-    else
-      assertNotCaptured "python-2-7-eol-faq";
-    fi
+    assertCaptured "python-2-7-eol-faq";
     assertNotCaptured "security update"
     assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
@@ -33,41 +29,29 @@ testPython2() {
 testPython2_warn() {
     compile "python2_warn"
     assertCaptured "python-2.7.15"
-    if [[ $(date "+%Y") > "2019" ]]; then
-      assertCaptured "python-2-7-eol-faq";
-    else
-      assertNotCaptured "python-2-7-eol-faq";
-    fi
+    assertCaptured "python-2-7-eol-faq";
     assertCaptured "Only the latest version"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
 }
 
-testPython2_fail() {
-    compile "python2_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
-}
-
 testPython3_4() {
   compile "python3_4"
   assertCaptured $LATEST_34
   assertNotCaptured "security update"
   assertCaptured "Installing pip 19.1.1, setuptools 43.0.0 and wheel 0.33.6"
-  assertCapturedSuccess
+  # Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
+  # and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
+  assertCapturedSuccessWithStdErr
 }
 
 testPython3_4_warn() {
   compile "python3_4_warn"
   assertCaptured "python-3.4.9"
   assertCaptured "security update!"
-  assertCapturedSuccess
-}
-
-testPython3_4_fail() {
-    compile "python3_4_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
+  # Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
+  # and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
+  assertCapturedSuccessWithStdErr
 }
 
 testPython3_5() {
@@ -86,12 +70,6 @@ testPython3_5_warn() {
   assertCapturedSuccess
 }
 
-testPython3_5_fail() {
-    compile "python3_5_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
-}
-
 testPython3_6() {
   updateVersion "python3_6" $LATEST_36
   compile "python3_6"
@@ -110,12 +88,6 @@ testPython3_6_warn() {
   assertCapturedSuccess
 }
 
-testPython3_6_fail() {
-    compile "python3_6_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
-}
-
 testPython3_7() {
   updateVersion "python3_7" $LATEST_37
   compile "python3_7"
@@ -142,24 +114,6 @@ testPython3_7_warn() {
   fi
 }
 
-testPython3_7_fail() {
-  compile "python3_7_fail"
-  assertCaptured "Aborting"
-  assertCapturedError
-}
-
-testPython3_7_warn() {
-  compile "python3_8_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "python-3.8.0"
-    assertCaptured "security update!"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
-  fi
-}
-
 testPython3_8() {
   updateVersion "python3_8" $LATEST_38
   compile "python3_8"
@@ -174,8 +128,49 @@ testPython3_8() {
   fi
 }
 
-testPython3_8_fail() {
-  compile "python3_8_fail"
+testPython3_8_warn() {
+  compile "python3_8_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "python-3.8.0"
+    assertCaptured "security update!"
+    assertCaptured "Installing SQLite3"
+    assertCapturedSuccess
+  fi
+}
+
+testPython3_9() {
+  # Cedar-14 is EOL, so we're not building new major Python versions for it.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
+  fi
+  compile "python3_9"
+  assertNotCaptured "security update"
+  assertCaptured $LATEST_39
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
+}
+
+testPython3_9_warn() {
+  # Cedar-14 is EOL, so we're not building new major Python versions for it.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
+  fi
+  # Can't test the version warning until there is at least one old version of Python 3.9.
+  if [[ "${LATEST_39}" = "python-3.9.0" ]]; then
+    return
+  fi
+  compile "python3_9_warn"
+  assertCaptured "python-3.9.0"
+  assertCaptured "security update!"
+  assertCapturedSuccess
+}
+
+testPythonVersionInvalid() {
+  compile "python_version_invalid"
+  assertCaptured "Requested runtime (python-3.8.99) is not available for this stack"
   assertCaptured "Aborting"
   assertCapturedError
 }
@@ -222,6 +217,30 @@ testPypy2_7_warn() {
   fi
 }
 
+testStickyPythonVersion() {
+  local cache_dir="$(mktmpdir)"
+  compile "python3_6_warn" "$cache_dir"
+  assertCaptured "Installing python-3.6.7"
+  assertCapturedSuccess
+  compile "no-runtime-txt" "$cache_dir"
+  assertCaptured "Installing python-3.6.7"
+  assertCapturedSuccess
+  # Whilst this file seems like an implementation detail (so something that should
+  # not be tested), we must guarantee the filename remains consistent for backwards
+  # compatibility across buildpack versions for already-built apps.
+  assertFile "python-3.6.7" ".heroku/python-version"
+}
+
+testPythonVersionChange() {
+  local cache_dir="$(mktmpdir)"
+  compile "python3_6_warn" "$cache_dir"
+  assertCaptured "Installing python-3.6.7"
+  assertCapturedSuccess
+  compile "python3_6" "$cache_dir"
+  assertCaptured "Found python-3.6.7, removing"
+  assertCapturedSuccess
+}
+
 pushd $(dirname 0) >/dev/null
 popd >/dev/null
 

test/utils

@@ -85,12 +85,20 @@ assertNotCaptured()
 assertCapturedSuccess()
 {
   assertEquals "Captured exit code -" "0" "${RETURN}"
-  # assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
+  assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
 
-  if [ $RETURN -ne 0 -a -z "$(cat ${STD_ERR})" ]; then
-    # Failing exit code but stderr was empty. Display stdout to help debugging.
-    cat $STD_OUT
-    echo
+  if [ $RETURN -ne 0 -o -n "$(cat ${STD_ERR})" ]; then
+    debug
+  fi
+}
+
+assertCapturedSuccessWithStdErr()
+{
+  assertEquals "Captured exit code -" "0" "${RETURN}"
+  assertNotEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
+
+  if [ ${RETURN} -ne 0 ]; then
+    debug
   fi
 }
 
@@ -151,9 +159,14 @@ _assertContains()
 
 debug()
 {
-  cat $STD_OUT
-  echo '^^^^^^'
-  cat $STD_ERR
+  echo
+  echo '### STD_OUT ###'
+  cat "${STD_OUT}"
+  echo
+  echo '### STD_ERR ###'
+  cat "${STD_ERR}"
+  echo
+  echo
 }
 
 assertContains()