Release v174 (#1023)

They are now displayed in the build output (instead of being sent to
`/dev/null`) and fail the build early instead of failing later in
`bin/steps/pip-install`.

Fixes #1002.

.github/workflows/check_changelog.yml

@@ -1,14 +1,17 @@
 name: Check Changelog
 
 on:
- pull_request:
-  types: [opened, reopened, edited, synchronize]
+  pull_request:
+    types: [opened, reopened, edited, synchronize]
 
 jobs:
- build:
-   runs-on: ubuntu-latest
-   steps:
-   - uses: actions/checkout@v1
-   - name: Check that CHANGELOG is touched
-     run: |
-       cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
+  check:
+    runs-on: ubuntu-latest
+    if: |
+      !contains(github.event.pull_request.body, '[skip changelog]') &&
+      !contains(github.event.pull_request.body, '[changelog skip]') &&
+      !contains(github.event.pull_request.body, '[skip ci]')
+    steps:
+      - uses: actions/checkout@v1
+      - name: Check that CHANGELOG is touched
+        run: git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md

.travis.yml

@@ -1,40 +1,40 @@
-language: ruby
+language: minimal
 dist: bionic
-sudo: required
 branches:
   only:
   - master
 
-rvm:
-- 2.6.6
-before_script:
-  - gem install bundler -v 1.16.2
-
 script:
-  - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
-  - docker run --rm -e "STACK=cedar-14" travis-build-cedar-14 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-  - docker run --rm -e "STACK=heroku-16" travis-build-heroku-16 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-  - docker run --rm -e "STACK=heroku-18" travis-build-heroku-18 bash $TESTFOLDER
+  - make test STACK="${STACK}" TEST_CMD="${TEST_CMD}"
 
 jobs:
   include:
-  - stage: Bash linting (shellcheck)
-    sudo: false
+  - name: Bash linting (shellcheck)
     script: make check
-  - stage: Hatchet Integration
+  - name: Hatchet integration tests
     if: env(TRAVIS_PULL_REQUEST_SLUG) = env(TRAVIS_REPO_SLUG)
-    name: Run Hatchet
+    language: ruby
+    rvm:
+      - 2.6.6
+    before_script:
+      - gem install bundler -v 1.16.2
     script:
       - bundle exec hatchet ci:setup
       - PARALLEL_SPLIT_TEST_PROCESSES=11 bundle exec parallel_split_test spec/hatchet/
 
 env:
-  matrix:
-    - TESTFOLDER=test/run-deps
-    - TESTFOLDER=test/run-versions
-    - TESTFOLDER=test/run-features
+  jobs:
+    - STACK=cedar-14 TEST_CMD=test/run-deps
+    - STACK=cedar-14 TEST_CMD=test/run-versions
+    - STACK=cedar-14 TEST_CMD=test/run-features
+
+    - STACK=heroku-16 TEST_CMD=test/run-deps
+    - STACK=heroku-16 TEST_CMD=test/run-versions
+    - STACK=heroku-16 TEST_CMD=test/run-features
+
+    - STACK=heroku-18 TEST_CMD=test/run-deps
+    - STACK=heroku-18 TEST_CMD=test/run-versions
+    - STACK=heroku-18 TEST_CMD=test/run-features
   global:
     - HATCHET_RETRIES=3
     - IS_RUNNING_ON_CI=true

CHANGELOG.md

@@ -1,5 +1,31 @@
 # Python Buildpack Changelog
 
+# Master
+
+
+# 174 (2020-07-30)
+
+- For repeat builds, also manage the installed versions of setuptools/wheel, rather than just that of pip (#1007).
+- Install an explicit version of wheel rather than the latest release at the time (#1007).
+- Output the installed version of pip, setuptools and wheel in the build log (#1007).
+- Errors installing pip/setuptools/wheel are now displayed in the build output and fail the build early (#1007).
+- Install pip using itself rather than `get-pip.py` (#1007).
+- Disable pip's version check + cache when installing pip/setuptools/wheel (#1007).
+- Install setuptools from PyPI rather than a vendored copy (#1007).
+- Reduce the number of environment variables exposed to `bin/{pre,post}_compile` and other subprocesses (#1011)
+
+# 173 (2020-07-21)
+
+- Python 3.8.5 is now available (CPython)
+
+# 172 (2020-07-17)
+
+- Python 3.8.4 is now available (CPython)
+
+# 171 (2020-07-07)
+
+- Python 3.6.11 and 3.7.8 are now available (CPython).
+
 # 170 (2020-05-19)
 
 - Python 2.7.18, 3.5.9, 3.7.7 and 3.8.3 are now available (CPython).

Gemfile.lock

@@ -1,12 +1,12 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.2.2)
+    activesupport (6.0.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-      zeitwerk (~> 2.2)
+      zeitwerk (~> 2.2, >= 2.2.2)
     concurrent-ruby (1.1.6)
     diff-lcs (1.3)
     erubis (2.7.0)
@@ -26,7 +26,7 @@ GEM
       threaded (~> 0)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.0)
+    minitest (5.14.1)
     minitest-retry (0.1.9)
       minitest (>= 5.0)
     moneta (1.0.0)
@@ -38,7 +38,7 @@ GEM
     platform-api (2.2.0)
       heroics (~> 0.0.25)
       moneta (~> 1.0.0)
-    rake (12.3.1)
+    rake (12.3.3)
     repl_runner (0.0.3)
       activesupport
     rrrretry (1.0.0)

Makefile

@@ -1,26 +1,26 @@
 # These targets are not files
-.PHONY: tests
+.PHONY: check test buildenv-heroku-16 buildenv-heroku-18 tools
 
-test: test-heroku-18 test-heroku-16 test-cedar-14
+STACK ?= heroku-18
+TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
+
+ifeq ($(STACK),cedar-14)
+	# Cedar-14 doesn't have a build image varient.
+	IMAGE_TAG := heroku/cedar:14
+else
+	# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
+	IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
+endif
 
 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
 	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
 	@shellcheck -x bin/steps/hooks/*
 
-test-cedar-14:
-	@echo "Running tests in docker (cedar-14)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
+test:
+	@echo "Running tests using: STACK=$(STACK) TEST_CMD='$(TEST_CMD)'"
 	@echo ""
-
-test-heroku-16:
-	@echo "Running tests in docker (heroku-16)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
-	@echo ""
-
-test-heroku-18:
-	@echo "Running tests in docker (heroku-18)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
+	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
 	@echo ""
 
 buildenv-heroku-16:

NOTICE

@@ -86,31 +86,6 @@ Each version is given a distinguishing version number. If the Library as you rec
 
 If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library.
 
-get-pip.py license
-------------------
-
-Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-
 pip-pop license
 ---------------
 
@@ -134,4 +109,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+THE SOFTWARE.

README.md

@@ -62,9 +62,9 @@ Specify a Python Runtime
 
 Supported runtime options include:
 
-- `python-3.8.3`
-- `python-3.7.7`
-- `python-3.6.10`
+- `python-3.8.5`
+- `python-3.7.8`
+- `python-3.6.11`
 - `python-2.7.18`
 
 ## Tests
@@ -72,17 +72,22 @@ Supported runtime options include:
 The buildpack tests use [Docker](https://www.docker.com/) to simulate
 Heroku's [stack images.](https://devcenter.heroku.com/articles/stack)
 
-To run the test suite:
+To run the test suite against the default stack:
 
 ```
 make test
 ```
 
-Or to test in a particular stack:
+Or to test against a particular stack:
 
 ```
-make test-heroku-18
-make test-heroku-16
+make test STACK=heroku-16
+```
+
+To run only a subset of the tests:
+
+```
+make test TEST_CMD=tests/versions
 ```
 
 The tests are run via the vendored

bin/compile

@@ -16,7 +16,9 @@
 set -eo pipefail
 
 # Boostrap the Buildpack Standard Library.
-export BPLOG_PREFIX="buildpack.python"
+# Disable unused env var warning since shellcheck doesn't know about the stdlib.
+# shellcheck disable=2034
+BPLOG_PREFIX="buildpack.python"
 export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
 
 [ "$BUILDPACK_XTRACE" ] && set -o xtrace
@@ -63,37 +65,13 @@ PYPY36="pypy3.6"
 
 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
-# If pip doesn't match this version (the version we install), run the installer.
-PIP_UPDATE="20.0.2"
-
-for file in "$BUILD_DIR/runtime.txt" "$CACHE_DIR/.heroku/python-version" ; do
-    [ -f "$file" ] || continue
-
-    version=$(tr -d '[:space:]' < "$file")
-
-    case "$version" in "$PY34"*)
-        # Python 3.4 support was dropped in pip >= 19.2.
-        PIP_UPDATE="19.1.1"
-        break
-        ;;
-    esac
-done
-
-if [[ -f "$BUILD_DIR/Pipfile" ]]; then
-    # Do not force pipenv users to re-install pipenv locally.
-    PIP_UPDATE="9.0.2"
-fi
-
-export DEFAULT_PYTHON_STACK PIP_UPDATE
-export PY37 PY36 PY35 PY27 PY34
 
 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.
 # The `warnings` subscript then greps through this for common problems and guides
 # the user towards resolution of known issues.
 WARNINGS_LOG=$(mktemp)
-export WARNINGS_LOG
-export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
+RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
 
 # The buildpack ships with a few executable tools (e.g. pip-grep, etc).
 # This installs them into the path, so we can execute them directly.
@@ -217,8 +195,6 @@ else
   CACHED_PYTHON_STACK=$STACK
 fi
 
-export CACHED_PYTHON_STACK
-
 # Pipenv Python version support.
 # Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version).
 # Convert it to a runtime.txt file.

bin/default_pythons

@@ -1,21 +1,16 @@
 #!/usr/bin/env bash
 
-DEFAULT_PYTHON_VERSION="python-3.6.10"
-LATEST_38="python-3.8.3"
-LATEST_37="python-3.7.7"
-LATEST_36="python-3.6.10"
+# Disable unused env var warning, since shellcheck doesn't take into account
+# that this file is sourced. We don't want to use export since it exposes
+# the env vars to subprocesses.
+# shellcheck disable=2034
+
+DEFAULT_PYTHON_VERSION="python-3.6.11"
+LATEST_38="python-3.8.5"
+LATEST_37="python-3.7.8"
+LATEST_36="python-3.6.11"
 LATEST_35="python-3.5.9"
 LATEST_34="python-3.4.10"
 LATEST_27="python-2.7.18"
 PYPY_36="pypy3.6-7.3.1"
 PYPY_27="pypy2.7-7.3.1"
-
-export DEFAULT_PYTHON_VERSION \
-    LATEST_38 \
-    LATEST_37 \
-    LATEST_36 \
-    LATEST_35 \
-    LATEST_34 \
-    LATEST_27 \
-    PYPY_36 \
-    PYPY_27

bin/steps/README.MD

@@ -2,13 +2,6 @@
 
 TODO: Add context on Python install steps, such as why symlinking vs copying
 
-## Installing the Pip tool
-
-The Python Buildpack uses a tool called `get-pip` to install the pip tool. This
-is done in the `python` script.
-
-This is in part because Python historically did not come with pip by default.
-
 ## Installing Python packages using Pip
 
 ### Convention: Use `python` process to invoke Pip

bin/steps/python

@@ -131,34 +131,52 @@ if [ ! "$SKIP_INSTALL" ]; then
   # Record for future reference.
   echo "$PYTHON_VERSION" > .heroku/python-version
   echo "$STACK" > .heroku/python-stack
-  FRESH_PYTHON=true
 
   hash -r
 fi
 
-# Heroku uses the get-pip utility maintained by the Python community to vendor Pip.
-# https://github.com/pypa/get-pip
-GETPIP_URL="https://lang-python.s3.amazonaws.com/etc/get-pip.py"
-GETPIP_PY="${TMPDIR:-/tmp}/get-pip.py"
-
-if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then
-    mcount "failure.python.get-pip"
-    echo "Failed to pull down get-pip"
-    exit 1
-fi
-
-# If a new Python has been installed or Pip isn't up to date:
-if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
-
-  puts-step "Installing pip"
-
-  # Remove old installations.
-  rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
-  rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
-
-  /app/.heroku/python/bin/python "$GETPIP_PY" pip=="$PIP_UPDATE" &> /dev/null
-  /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
-fi
-
 set -e
+
+PIP_VERSION='20.0.2'
+SETUPTOOLS_VERSION='39.0.1'
+WHEEL_VERSION='0.34.2'
+
+if [[ "${PYTHON_VERSION}" == ${PY34}* ]]; then
+  # Python 3.4 support was dropped in pip 19.2+ and wheel 0.34.0+.
+  PIP_VERSION='19.1.1'
+  WHEEL_VERSION='0.33.6'
+fi
+
+# We don't use get-pip.py, since:
+#  - it uses `--force-reinstall`, which is unnecessary here and slows down repeat builds
+#  - it means downloading pip twice (once embedded in get-pip.py, and again during
+#    the install, since get-pip.py can't install the embedded version directly)
+#  - we would still have to manage several versions of get-pip.py, to support older Pythons.
+# Instead, we use the pip wheel to install itself, using the method described here:
+# https://github.com/pypa/pip/issues/2351#issuecomment-69994524
+PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl"
+PIP_WHEEL_URL="https://lang-python.s3.amazonaws.com/common/${PIP_WHEEL_FILENAME}"
+PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}"
+
+if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then
+  mcount "failure.python.download-pip"
+  puts-warn "Failed to download pip"
+  exit 1
+fi
+
+if [[ -f "$BUILD_DIR/Pipfile" ]]; then
+  # The buildpack is pinned to old pipenv, which requires older pip.
+  # Pip 9.0.2 doesn't support installing itself from a wheel, so we have to use split
+  # versions here (ie: installer pip version different from target pip version).
+  PIP_VERSION='9.0.2'
+  PIP_TO_INSTALL="pip==${PIP_VERSION}"
+else
+  PIP_TO_INSTALL="${PIP_WHEEL}"
+fi
+
+puts-step "Installing pip ${PIP_VERSION}, setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}"
+
+/app/.heroku/python/bin/python "${PIP_WHEEL}/pip" install --quiet --disable-pip-version-check --no-cache \
+  "${PIP_TO_INSTALL}" "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}"
+
 hash -r

builds/runtimes/python-3.6.11

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.7.8

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.4

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.5

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

test/fixtures/hooks/bin/post_compile

@@ -0,0 +1,4 @@
+set -euo pipefail
+
+echo "post_compile ran!"
+echo "post_compile env: $(printenv | cut -d '=' -f 1 | sort | xargs)."

test/fixtures/hooks/bin/pre_compile

@@ -0,0 +1,4 @@
+set -euo pipefail
+
+echo "pre_compile ran!"
+echo "pre_compile env: $(printenv | cut -d '=' -f 1 | sort | xargs)."

test/run-deps

@@ -5,8 +5,9 @@
 source "bin/default_pythons"
 
 testAirflow() {
-  export SLUGIFY_USES_TEXT_UNIDECODE="yes"
-  compile "airflow"
+  local env_dir="$(mktmpdir)"
+  echo 'yes' > "${env_dir}/SLUGIFY_USES_TEXT_UNIDECODE"
+  compile 'airflow' '' "${env_dir}"
   assertCaptured "apache-airflow==1.10.2"
   assertCapturedSuccess
 }
@@ -17,27 +18,30 @@ testCollectstatic() {
 }
 
 testGEOS() {
-  export BUILD_WITH_GEO_LIBRARIES=1
-  compile "geos"
+  local env_dir="$(mktmpdir)"
+  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
+  compile 'geos' '' "${env_dir}"
   assertCaptured "geos"
   assertCapturedSuccess
 }
 
 testGEOSDeprecation() {
-  export BUILD_WITH_GEO_LIBRARIES=1
-  compile "geos"
+  local env_dir="$(mktmpdir)"
+  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
+  compile 'geos' '' "${env_dir}"
   assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated.
  !     An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
   assertCapturedSuccess
 }
 
 testNLTK() {
+  local env_dir="$(mktmpdir)"
   # NOTE: This is a RuntimeWarning emitted by Python 3's runpy.py script
   # which is what is used when you call `python -m <module>`. This is due to
   # how nltk imports things. It's not actually an error, but it would probably
   # be bad to silence in Production.
-  export PYTHONWARNINGS="ignore::RuntimeWarning"
-  compile "nltk"
+  echo 'ignore::RuntimeWarning' > "${env_dir}/PYTHONWARNINGS"
+  compile 'nltk' '' "${env_dir}"
   assertCaptured "[nltk_data] Downloading package city_database" "STD_ERR"
   assertCapturedSuccess
 }

test/run-features

@@ -44,6 +44,7 @@ testStandardRequirements() {
 
 testPipenv() {
   compile "pipenv"
+  assertCaptured "Installing pip 9.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCapturedSuccess
 }
 
@@ -86,6 +87,49 @@ testDontWarnOldDjango() {
   assertCapturedSuccess
 }
 
+testHooks() {
+  # Test that the hooks are called correctly, and that the environment contains
+  # the app's config vars but no unexpected env vars from the buildpack.
+  local env_dir="$(mktmpdir)"
+  echo 'test' > "${env_dir}/SOME_APP_CONFIG_VAR"
+  local expected_env_vars=(
+    _
+    BIN_DIR
+    BUILD_DIR
+    BUILDPACK_LOG_FILE
+    CACHE_DIR
+    C_INCLUDE_PATH
+    CPLUS_INCLUDE_PATH
+    ENV_DIR
+    EXPORT_PATH
+    HOME
+    LANG
+    LD_LIBRARY_PATH
+    LIBRARY_PATH
+    OLDPWD
+    PATH
+    PKG_CONFIG_PATH
+    PROFILE_PATH
+    PWD
+    PYTHONUNBUFFERED
+    SHLVL
+    SOME_APP_CONFIG_VAR
+    STACK
+    VENDOR_URL
+  )
+  if [[ "${STACK}" == "cedar-14" || "${STACK}" == "heroku-16" ]]; then
+    # Remove "OLDPWD" from expected_env_vars since for bash <4.4 it's not exported to subshells:
+    # https://github.com/heroku/heroku-buildpack-python/pull/1011#issuecomment-665117835
+    read -ra expected_env_vars <<< "${expected_env_vars[@]/OLDPWD/}"
+  fi
+  compile 'hooks' '' "${env_dir}"
+  assertCaptured "pre_compile ran!"
+  assertCaptured "pre_compile env: ${expected_env_vars[*]}."
+  assertCaptured "post_compile ran!"
+  assertCaptured "post_compile env: ${expected_env_vars[*]}."
+  assertCapturedSuccess
+}
+
 pushd $(dirname 0) >/dev/null
 popd >/dev/null
 

test/run-versions

@@ -9,6 +9,7 @@ testPythonDefault() {
   compile "pythonDefault"
   assertCaptured $DEFAULT_PYTHON_VERSION
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -24,6 +25,7 @@ testPython2() {
       assertNotCaptured "python-2-7-eol-faq";
     fi
     assertNotCaptured "security update"
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
 }
@@ -51,6 +53,7 @@ testPython3_4() {
   compile "python3_4"
   assertCaptured $LATEST_34
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 19.1.1, setuptools 39.0.1 and wheel 0.33.6"
   # if cedar 14 and legacy binaries, fail. if cedar 14 and staging, succeed.
   if [[ ! -n $USE_STAGING_BINARIES ]] && [[ $STACK == "cedar-14" ]]; then
       assertCapturedError
@@ -86,6 +89,7 @@ testPython3_5() {
   compile "python3_5"
   assertCaptured $LATEST_35
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -108,6 +112,7 @@ testPython3_6() {
   compile "python3_6"
   assertCaptured $LATEST_36
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -134,6 +139,7 @@ testPython3_7() {
   else
     assertNotCaptured "security update"
     assertCaptured $LATEST_37
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
   fi
@@ -177,6 +183,7 @@ testPython3_8() {
   else
     assertNotCaptured "security update"
     assertCaptured $LATEST_38
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
   fi
@@ -195,6 +202,7 @@ testPypy3_6() {
   else
     assertCaptured "Installing pypy"
     assertCaptured "$PYPY_36"
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCapturedSuccess
   fi
 }
@@ -206,6 +214,7 @@ testPypy2_7() {
   else
     assertCaptured "Installing pypy"
     assertCaptured "$PYPY_27"
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCapturedSuccess
   fi
 }

test/utils

@@ -56,21 +56,6 @@ resetCapture()
   unset rtrn # deprecated
 }
 
-detect()
-{
-  capture ${BUILDPACK_HOME}/bin/detect ${BUILD_DIR}
-}
-
-compile()
-{
-  capture ${BUILDPACK_HOME}/bin/compile ${BUILD_DIR} ${CACHE_DIR}
-}
-
-release()
-{
-  capture ${BUILDPACK_HOME}/bin/release ${BUILD_DIR}
-}
-
 updateVersion()
 {
   echo "$2" > "test/fixtures/${1}/runtime.txt"
@@ -234,13 +219,18 @@ default_process_types_cleanup() {
   fi
 }
 
+run_in_clean_env() {
+  # Prevent stray environment variables from outside the test runner being exposed to tests.
+  env -i HOME="${HOME}" LANG="${LANG}" PATH="${PATH}" STACK="${STACK}" "$@"
+}
+
 compile() {
   default_process_types_cleanup
   bp_dir=$(mktmpdir)
   compile_dir=$(mktmpdir)
   cp -a $(pwd)/* ${bp_dir}
   cp -a ${bp_dir}/test/fixtures/$1/. ${compile_dir}
-  capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3
+  capture run_in_clean_env ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3
 }
 
 compileDir() {
@@ -252,13 +242,13 @@ compileDir() {
   local env_dir=$3
 
   cp -a $(pwd)/* ${bp_dir}
-  capture ${bp_dir}/bin/compile ${compile_dir} ${cache_dir} ${env_dir}
+  capture run_in_clean_env ${bp_dir}/bin/compile ${compile_dir} ${cache_dir} ${env_dir}
 }
 
 release() {
   bp_dir=$(mktmpdir)
   cp -a $(pwd)/* ${bp_dir}
-  capture ${bp_dir}/bin/release ${bp_dir}/test/fixtures/$1
+  capture run_in_clean_env ${bp_dir}/bin/release ${bp_dir}/test/fixtures/$1
 }
 
 assertFile() {

tests.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-if [[ ! "$STACK" ]]; then
-    echo '$STACK must be set! (heroku-16 | cedar-14)'
-    exit 1
-fi
-
-if [[ "$STACK" == "cedar-14" ]]; then
-    make test-cedar-14
-    exit $?
-fi
-
-if [[ "$STACK" == "heroku-16" ]]; then
-    make test-heroku-16
-    exit $?
-fi
-
-if [[ "$STACK" == "heroku-18" ]]; then
-    make test-heroku-18
-    exit $?
-fi