Release v180 (#1078 )

Closes @W-8120452@.
Add support for Python 3.8.6 (#1072 )
2026-06-05 23:10:16 +00:00 · 2020-09-24 19:17:18 +01:00 · 2020-09-24 18:55:27 +01:00 · 2020-09-23 14:49:46 +01:00 · 2020-09-23 11:15:54 +01:00 · 2020-09-22 18:48:31 +01:00
87 changed files with 798 additions and 866 deletions
@@ -0,0 +1,6 @@
+{
+  "productTag": "a1aB0000000MR0RIAW",
+  "issueTypeLabels": { "gus: story": "USER STORY", "gus: bug": "BUG P3" },
+  "defaultBuild": "Heroku Unscheduled",
+  "statusWhenClosed": "CLOSED"
+}
@@ -1 +1 @@
-* @heroku/languages
+* @heroku/languages
@@ -0,0 +1,15 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "c: dependencies"
+
+  - package-ecosystem: "bundler"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "c: dependencies"
@@ -1,12 +1,17 @@
 name: Check Changelog

-on: [pull_request]
+on:
+  pull_request:
+    types: [opened, reopened, edited, synchronize]

 jobs:
- build:
-   runs-on: ubuntu-latest
-   steps:
-   - uses: actions/checkout@v1
-   - name: Check that CHANGELOG is touched
-     run: |
-       cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
+  check:
+    runs-on: ubuntu-latest
+    if: |
+      !contains(github.event.pull_request.body, '[skip changelog]') &&
+      !contains(github.event.pull_request.body, '[changelog skip]') &&
+      !contains(github.event.pull_request.body, '[skip ci]')
+    steps:
+      - uses: actions/checkout@v1
+      - name: Check that CHANGELOG is touched
+        run: git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
@@ -10,3 +10,5 @@ buildpack/*

 builds/dockerenv.staging*
 builds/dockerenv.production
+
+test/scratch
@@ -1,7 +0,0 @@
-repos:
-   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
-    hooks:
-    -   id: check-yaml
-    -   id: end-of-file-fixer
-    -   id: trailing-whitespace
@@ -1,39 +1,41 @@
-language: ruby
-dist: trusty
-sudo: required
-
-rvm:
- 2.4.4
-before_script:
-  - gem install bundler -v 1.16.2
-  - bundle exec hatchet ci:setup
+language: minimal
+dist: focal
+branches:
+  only:
+  - main

 script:
-  - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
-  - docker run --rm -e "STACK=cedar-14" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-cedar-14 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-  - docker run --rm -e "STACK=heroku-16" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-16 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-  - docker run --rm -e "STACK=heroku-18" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-18 bash $TESTFOLDER
+  - make test STACK="${STACK}" TEST_CMD="${TEST_CMD}"

 jobs:
  include:
-  - stage: Bash linting (shellcheck)
-    sudo: false
+  - name: Bash linting (shellcheck)
    script: make check
-  - stage: Hatchet Integration
-    if: branch = master
-    name: Run Hatchet
-    script: "bundle exec rspec"
+  - name: Hatchet integration tests
+    if: env(HEROKU_API_USER) IS present AND env(HEROKU_API_KEY) IS present
+    language: ruby
+    rvm:
+      - 2.7
+    before_script:
+      - bundle exec hatchet ci:setup
+    script:
+      - PARALLEL_SPLIT_TEST_PROCESSES=11 bundle exec parallel_split_test spec/hatchet/
+
 env:
-  matrix:
-    - TESTFOLDER=test/run-deps
-    - TESTFOLDER=test/run-versions
-    - TESTFOLDER=test/run-features
+  jobs:
+    - STACK=cedar-14 TEST_CMD=test/run-deps
+    - STACK=cedar-14 TEST_CMD=test/run-versions
+    - STACK=cedar-14 TEST_CMD=test/run-features
+
+    - STACK=heroku-16 TEST_CMD=test/run-deps
+    - STACK=heroku-16 TEST_CMD=test/run-versions
+    - STACK=heroku-16 TEST_CMD=test/run-features
+
+    - STACK=heroku-18 TEST_CMD=test/run-deps
+    - STACK=heroku-18 TEST_CMD=test/run-versions
+    - STACK=heroku-18 TEST_CMD=test/run-features
  global:
    - HATCHET_RETRIES=3
    - IS_RUNNING_ON_CI=true
-    - HATCHET_APP_LIMIT=5
+    - HATCHET_APP_LIMIT=80
    - HATCHET_DEPLOY_STRATEGY=git
-    - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
-    - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=
@@ -1,40 +1,130 @@
 # Python Buildpack Changelog

-# Master
+## Unreleased
+
+
+## v180 (2020-09-24)
+
+- Python 3.8.6 is now available (CPython) (#1072).
+
+## v179 (2020-09-23)
+
+- Remove duplicate pipenv metric event (#1070).
+- Emit metrics for how the Python version was chosen for an app (#1069).
+- Emit Python version metric events for all builds, not just clean installs (#1066).
+
+## v178 (2020-09-07)
+
+- Python 3.5.10 is now available (CPython) (#1062).
+
+## v177 (2020-08-18)
+
+- Python 3.6.12 and 3.7.9 are now available (CPython) (#1054).
+- The default Python version for new apps is now 3.6.12 (previously 3.6.11) (#1054).
+
+## v176 (2020-08-12)
+
+- Rebuild the Python 3.4.10 archives with the correct version of Python (#1048).
+- Fix the security update version check message for apps using PyPy (#1040).
+- Remove `vendor/test-utils` (#1043).
+
+## v175 (2020-08-05)
+
+- Update pip from 20.0.2 to 20.1.1 for Python 2.7 and Python 3.5+ (#1030).
+- Update setuptools from 39.0.1 to: (#1024)
+  - 44.1.1 for Python 2.7
+  - 43.0.0 for Python 3.4
+  - 47.1.1 for Python 3.5+
+- Switch the `heroku-buildpack-python` repository default branch from `master` to `main` (#1029).
+
+## v174 (2020-07-30)
+
+- For repeat builds, also manage the installed versions of setuptools/wheel, rather than just that of pip (#1007).
+- Install an explicit version of wheel rather than the latest release at the time (#1007).
+- Output the installed version of pip, setuptools and wheel in the build log (#1007).
+- Errors installing pip/setuptools/wheel are now displayed in the build output and fail the build early (#1007).
+- Install pip using itself rather than `get-pip.py` (#1007).
+- Disable pip's version check + cache when installing pip/setuptools/wheel (#1007).
+- Install setuptools from PyPI rather than a vendored copy (#1007).
+- Reduce the number of environment variables exposed to `bin/{pre,post}_compile` and other subprocesses (#1011).
+
+## v173 (2020-07-21)
+
+- Python 3.8.5 is now available (CPython).
+
+## v172 (2020-07-17)
+
+- Python 3.8.4 is now available (CPython).
+
+## v171 (2020-07-07)
+
+- Python 3.6.11 and 3.7.8 are now available (CPython).
+
+## v170 (2020-05-19)
+
+- Python 2.7.18, 3.5.9, 3.7.7 and 3.8.3 are now available (CPython).
+- PyPy 2.7 and 3.6, version 7.3.1 are now available (Note: PyPy support is in beta).
+- Docs: Fix explanation of runtime.txt generation when using pipenv.
+- Bugfix: Correctly detect Python version when using a `python_version` of `3.8` in `Pipfile.lock`.
+
+## v169 (2020-04-22)
+
+- Add a Hatchet test for python 3.8.2
+- Set Code Owners to @heroku/langauges
+- Bugfix: Caching on subsequent redeploys
+- Update tests to support latest version of Python
+
+## v168 (2020-04-06)
+
+- Doc: Update Readme with version numbers
+- update Code Owners to include the Heroku Buildpack Maintainers team
+- Deprecation warning: `BUILD_WITH_GEO_LIBRARIES` is now deprecated. See warning for details.
+- Clean up build log output
+- Update Python versions in README to match docs
+- Django version detection fixed, link updated
+
+## v167 (2020-03-26)
+
+- Add failcase for cache busting
+- Bugfix: Clearing pip dependencies
+
+## v166 (2020-03-05)
+
+- Correct ftp to https in vendored file
+- Warn for Django 1.11 approaching EOL, provide link to roadmap
+
+## v165 (2020-02-27)

 - Python 3.8.2 now available.

-
--------------------------------------------------------------------------------
-
-# 163 (2020-02-20)
+## v164 (2020-02-20)

 - Update requirements.txt builds to use Pip 20.0.2
 - Download get-pip.py to tmpdir instead of root dir

-# 163 (2019-12-23)
+## v163 (2019-12-23)

 - New pythons released:
  Python 3.8.1, 3.7.6, 3.6.10 (CPython)
  Beta Release: Pypy 2.7 and 3.6, version 7.2.0

-# 162 (2019-12-06)
+## v162 (2019-12-06)

 - Bug fix: fragile sqlite3 install

-# 161 (2019-12-2)
+## v161 (2019-12-2)

 - Bug fix: Sqlite3 version bump

-# 160 (2019-10-23)
+## v160 (2019-10-23)

 - Bugfix: Pipenv no longer installs twice in CI

-# 159 (2019-10-22)
+## v159 (2019-10-22)

 - Python 2.7.17 now available on Heroku 18 and 16.

-# 158 (2019-10-21)
+## v158 (2019-10-21)

 - Python 3.7.5 and 3.8.0 now available on Heroku 18 and 16.
 - Add support for Python 3.8 branch
@@ -42,11 +132,11 @@
  - Test Improvements
 - Add support for staging binary testing

-# 157 (2019-09-18)
+## v157 (2019-09-18)

 - Typo fixes

-# 156 (2019-09-12)
+## v156 (2019-09-12)

 - Python 3.6.9 and 3.7.4 now available.

@@ -54,125 +144,125 @@
 - Build utility and documentation updates
 - Bump Hatchet tests to point at new default python version.

-# 155 (2019-08-22)
+## v155 (2019-08-22)

 add docs and make target for heroku-18 bob builds

-# 154 (2019-07-17)
+## v154 (2019-07-17)

 Fix python 3.5.7 formula actually building 3.7.2

-# 153 (2019-06-21)
+## v153 (2019-06-21)

 Hotfix for broken heroku-16 deploys

-# 152 (2019-04-04)
+## v152 (2019-04-04)

 Python 3.7.3 now available.

-# 151 (2019-03-21)
+## v151 (2019-03-21)

 Python 3.5.7 and 3.4.10 now available on all Heroku stacks.

-# 150 (2019-03-13)
+## v150 (2019-03-13)

 Python 2.7.16 now available on all Heroku stacks.

-# 149 (2019-03-04)
+## v149 (2019-03-04)

 Hotfix for broken Cedar 14 deploys

-# 148 (2019-02-21)
+## v148 (2019-02-21)

 No user facing changes, improving internal metrics

-# 147 (2019-02-07)
+## v147 (2019-02-07)

 Python 3.7.2 and 3.6.8 now available on all Heroku stacks.

-# 146 (2018-11-11)
+## v146 (2018-11-11)

 Python 3.7.1, 3.6.7, 3.5.6 and 3.4.9 now available on all Heroku stacks.

-# 145 (2018-11-08)
+## v145 (2018-11-08)

 Testing and tooling expanded to better support new runtimes

-# 144 (2018-10-10)
+## v144 (2018-10-10)

 Switch to cautious upgrade for Pipenv install to ensure the pinned pip version
 is used with Pipenv

-# 143 (2018-10-09)
+## v143 (2018-10-09)

-Add support for detecting SLUGIFY_USES_TEXT_UNIDECODE, which is required to
+Add support for detecting `SLUGIFY_USES_TEXT_UNIDECODE`, which is required to
 install Apache Airflow version 1.10 or higher.

-# 142 (2018-10-08)
+## v142 (2018-10-08)

 Improvements to Python install messaging

-# 139, 140, 141
+## v139, 140, 141

 No user-facing changes, documenting for version clarity

-# 138 (2018-08-01)
+## v138 (2018-08-01)

 Use stack image SQLite3 instead of vendoring

-# 137 (2018-07-17)
+## v137 (2018-07-17)

 Prevent 3.7.0 from appearing as unsupported in buildpack messaging.

-# 136 (2018-06-28)
+## v136 (2018-06-28)

 Upgrade to 3.6.6 and support 3.7.0 on all runtimes.

-# 135 (2018-05-29)
+## v135 (2018-05-29)

 Upgrade Pipenv to v2018.5.18.

-# 134 (2018-05-02)
+## v134 (2018-05-02)

 Default to 3.6.5, bugfixes.

-# 133
+## v133

 Fixes for Pip 10 release.

-# 132
+## v132

 Improve pip installation, with the release of v9.0.2.

-# 131
+## v131

 Fix bug with pip.

-# 130
+## v130

 Better upgrade strategy for pip.

-# 129
+## v129

 Don't upgrade pip (from v128).

-# 128
+## v128

 Upgrade pip, pin to Pipenv v11.8.2.

-# 127
+## v127

 Pin to Pipenv v11.7.1.

-# 126
+## v126

 Bugfixes.

-# 125
+## v125

 Bugfixes.

-# 124
+## v124

 Update buildpack to automatically install `[dev-packages]` during Heroku CI Pipenv builds.

@@ -181,101 +271,101 @@ Update buildpack to automatically install `[dev-packages]` during Heroku CI Pipe
 - No longer warn if there is no `Procfile`.
 - Update Pipenv's "3.6" runtime specifier to point to "3.6.4".

-# 123
+## v123

 Update gunicorn `init.d` script to allow overrides.

-# 122
+## v122

 Update default Python to v3.6.4.

-# 121
+## v121

 Update default Python to v3.6.3.

-# 120
+## v120

 Use `$ pipenv --deploy`.

-# 119
+## v119

 Improvements to Pipenv support, warning on unsupported Python versions.

 - We now warn when a user is not using latest 2.x or 3.x Python.
 - Heroku now supports `[requires]` `python_full_version` in addition to `python_version`.

-# 118
+## v118

 Improvements to Pipenv support.

-# 117
+## v117

 Bug fix.

-# 116
+## v116

 Vendoring improvements.

 - Geos libraries should work on Heroku-16 now.
 - The libffi/libmemcached vendoring step is now skipped on Heroku-16 (since they are installed in the base image).

-# 115
+## v115

 Revert a pull request.

 - No longer using `sub_env` for `pip install` step.

-# 114
+## v114

 - Bugfixes.

 Blacklisting `PYTHONHOME` and `PYTHONPATH` for older apps. Upgrades to nltk support.

-# 113
+## v113

 Updates to Pipenv support.

-# 112
+## v112

 Bugfix.

 - Fixed grep output bug.

-# 111
+## v111

 Linting, bugfixes.

-# 110
+## v110

 Update default Python to 3.6.2.

-# 109
+## v109

 Update Default Python to 3.6.1, bugfixes.

 - Fixed automatic pip uninstall of dependencies removed from requirements.txt.

-# 108
+## v108

 Fix output for collectstatic step.

-# 107
+## v107

 Bugfix for C dependency installation.

-# 106
+## v106

- Don't install packages that could mess up packaging.
+Don't install packages that could mess up packaging.

 - The Python buildpack will automatically remove `six`, `pyparsing`, `appdirs`,
   `setuptools`, and `distribute` from a `requirements.txt` file now, as these
   packages are provided by the Python buildpack.

- # 105
+## v105

- Improvements to output messaging.
+Improvements to output messaging.

-# 104
+## v104

 General improvements.

@@ -285,52 +375,52 @@ General improvements.

 [395]: https://github.com/heroku/heroku-buildpack-python/pull/395

-## 103
+## v103

 Bug fixes and improvements.

 - Fix for Pipenv.
 - Fix for Heroku CI.
- Improve handling of WEB_CONCURRENCY when using multiple buildpacks.
- Adjust environment variables set during the build to more closely match those in the dyno environment (DYNO is now available, STACK is not).
+- Improve handling of `WEB_CONCURRENCY` when using multiple buildpacks.
+- Adjust environment variables set during the build to more closely match those in the dyno environment (`DYNO` is now available, `STACK` is not).
 - Restore the build cache prior to running bin/pre_compile.

-## 102
+## v102

 Buildpack code cleanup.

 - Improved messaging around NLTK.

-## 101
+## v101

 Updated setuptools installation method.

 - Improved pipenv support.

-## 100
+## v100

 Preliminary pipenv support.

-## 99
+## v99

 Cleanup.

-## 98
+## v98

 Official NLTK support and other improvements.

 - Support for `nltk.txt` file for declaring corpora to be downloaded.
- Leading zeros for auto-set WEB_CONCURRENCY.
+- Leading zeros for auto-set `WEB_CONCURRENCY`.

-## 97
+## v97

 Improved egg-link functionality.

-## 96
+## v96

 Bugfix.

-## 95
+## v95

 Improved output support.

@@ -366,7 +456,7 @@ Fixed bug with editable pip installations.

 Updated default Python 2.7.13.

- Python 2.7.13 uses UCS-4 build, more compatibile with linux wheels.
+- Python 2.7.13 uses UCS-4 build, more compatible with linux wheels.
 - Updated setuptools to v32.1.0.

 ## v86
@@ -392,7 +482,7 @@ Support for Heroku CI.

 ## v82 (2016-08-22)

-Update to library detection mechnisms (pip-pop).
+Update to library detection mechanisms (pip-pop).

 - Updated setuptools to v25.5.0

@@ -466,7 +556,7 @@ Revert to v66.

 ## v68 (2015-10-12)

-Fixed .heroku/venv error with modern apps.
+Fixed `.heroku/venv` error with modern apps.

 ## v67 (2015-10-12)

@@ -491,7 +581,6 @@ Updated Pip and Setuptools.
 - Setuptools updated to v18.3.2
 - Pip updated to v7.1.2

-
 ## v62 (2015-08-07)

 Updated Pip and Setuptools.
@@ -4,3 +4,5 @@ gem "rspec"
 gem "heroku_hatchet"
 gem "rspec-retry"
 gem "rake"
+gem "parallel_split_test"
+
@@ -1,71 +1,60 @@
 GEM
  remote: https://rubygems.org/
  specs:
-    activesupport (5.2.1)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-    concurrent-ruby (1.1.3)
-    diff-lcs (1.3)
+    diff-lcs (1.4.4)
    erubis (2.7.0)
-    excon (0.62.0)
-    heroics (0.0.25)
+    excon (0.76.0)
+    heroics (0.1.1)
      erubis (~> 2.0)
      excon
      moneta
      multi_json (>= 1.9.2)
-    heroku_hatchet (4.0.6)
+    heroku_hatchet (7.2.0)
      excon (~> 0)
-      minitest-retry (~> 0.1.9)
-      platform-api (~> 2)
-      repl_runner (~> 0.0.3)
+      platform-api (~> 3)
      rrrretry (~> 1)
      thor (~> 0)
      threaded (~> 0)
-    i18n (1.1.1)
-      concurrent-ruby (~> 1.0)
-    minitest (5.11.3)
-    minitest-retry (0.1.9)
-      minitest (>= 5.0)
    moneta (1.0.0)
-    multi_json (1.13.1)
-    platform-api (2.2.0)
-      heroics (~> 0.0.25)
+    multi_json (1.15.0)
+    parallel (1.19.2)
+    parallel_split_test (0.8.0)
+      parallel (>= 0.5.13)
+      rspec (>= 3.1.0)
+    platform-api (3.0.0)
+      heroics (~> 0.1.1)
      moneta (~> 1.0.0)
-    rake (12.3.1)
-    repl_runner (0.0.3)
-      activesupport
+      rate_throttle_client (~> 0.1.0)
+    rake (13.0.1)
+    rate_throttle_client (0.1.2)
    rrrretry (1.0.0)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-retry (0.6.1)
+      rspec-support (~> 3.9.0)
+    rspec-retry (0.6.2)
      rspec-core (> 3.3)
-    rspec-support (3.8.0)
+    rspec-support (3.9.3)
    thor (0.20.3)
-    thread_safe (0.3.6)
    threaded (0.0.4)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)

 PLATFORMS
  ruby

 DEPENDENCIES
  heroku_hatchet
+  parallel_split_test
  rake
  rspec
  rspec-retry

 BUNDLED WITH
-   1.16.3
+   2.1.4
@@ -1,26 +1,26 @@
 # These targets are not files
-.PHONY: tests
+.PHONY: check test buildenv-heroku-16 buildenv-heroku-18 tools

-test: test-heroku-18 test-heroku-16 test-cedar-14
+STACK ?= heroku-18
+TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
+
+ifeq ($(STACK),cedar-14)
+	# Cedar-14 doesn't have a build image varient.
+	IMAGE_TAG := heroku/cedar:14
+else
+	# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
+	IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
+endif

 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
 	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
 	@shellcheck -x bin/steps/hooks/*

-test-cedar-14:
-	@echo "Running tests in docker (cedar-14)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
+test:
+	@echo "Running tests using: STACK=$(STACK) TEST_CMD='$(TEST_CMD)'"
 	@echo ""
-
-test-heroku-16:
-	@echo "Running tests in docker (heroku-16)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
-	@echo ""
-
-test-heroku-18:
-	@echo "Running tests in docker (heroku-18)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
+	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
 	@echo ""

 buildenv-heroku-16:
@@ -86,31 +86,6 @@ Each version is given a distinguishing version number. If the Library as you rec

 If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library.

-get-pip.py license
------------------
-
-Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-
 pip-pop license
 ---------------

@@ -134,4 +109,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+THE SOFTWARE.
@@ -1,9 +0,0 @@
-[[source]]
-url = "https://pypi.python.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[dev-packages]
-
-[packages]
-"bob-builder" = "==0.0.13"
@@ -1,52 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "36d17c46a8b1b844b3cae475f6f42d6c0a9e59b2a9685cbcdc0985656a7a129f"
-        },
-        "host-environment-markers": {
-            "implementation_name": "cpython",
-            "implementation_version": "3.6.3",
-            "os_name": "posix",
-            "platform_machine": "x86_64",
-            "platform_python_implementation": "CPython",
-            "platform_release": "16.7.0",
-            "platform_system": "Darwin",
-            "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64",
-            "python_full_version": "3.6.3",
-            "python_version": "3.6",
-            "sys_platform": "darwin"
-        },
-        "pipfile-spec": 6,
-        "requires": {},
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.python.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "bob-builder": {
-            "hashes": [
-                "sha256:b4de49a8e436fcaf82236ea43f78413b4a4c92100726e382ab57b6bdfb38fe64",
-                "sha256:288e3e765c4890fe9a63ae52ac6b4a963c13fe508482c70ff701a5ae21b9a673"
-            ],
-            "version": "==0.0.13"
-        },
-        "boto": {
-            "hashes": [
-                "sha256:13be844158d1bd80a94c972c806ec8381b9ea72035aa06123c5db6bc6a6f3ead",
-                "sha256:deb8925b734b109679e3de65856018996338758f4b916ff4fe7bb62b6d7000d1"
-            ],
-            "version": "==2.48.0"
-        },
-        "docopt": {
-            "hashes": [
-                "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
-            ],
-            "version": "==0.6.2"
-        }
-    },
-    "develop": {}
-}
@@ -2,7 +2,7 @@

 # Heroku Buildpack: Python

-[![Build Status](https://travis-ci.org/heroku/heroku-buildpack-python.svg?branch=master)](https://travis-ci.org/heroku/heroku-buildpack-python)
+[![Build Status](https://travis-ci.com/heroku/heroku-buildpack-python.svg?branch=main)](https://travis-ci.com/heroku/heroku-buildpack-python)

 This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/buildpacks) for Python apps.

@@ -16,7 +16,7 @@ See it in Action
 $ ls
 my-application		requirements.txt	runtime.txt

-$ git push heroku master
+$ git push heroku main
 Counting objects: 4, done.
 Delta compression using up to 8 threads.
 Compressing objects: 100% (2/2), done.
@@ -26,7 +26,7 @@ remote: Compressing source files... done.
 remote: Building source:
 remote:
 remote: -----> Python app detected
-remote: -----> Installing python-3.7.4
+remote: -----> Installing python
 remote: -----> Installing pip
 remote: -----> Installing SQLite3
 remote: -----> Installing requirements with pip
@@ -44,9 +44,10 @@ A `requirements.txt` must be present at the root of your application's repositor

 To specify your python version, you also need a `runtime.txt` file - unless you are using the default Python runtime version.

-Current default Python Runtime: Python 3.6.9
+Current default Python Runtime: Python 3.6.12

-Alternatively, you can provide a `setup.py` file, or a `Pipfile`. Using `Pipenv` will generate `runtime.txt` based on `python-version` at build time.
+Alternatively, you can provide a `setup.py` file, or a `Pipfile`.
+Using `pipenv` will generate `runtime.txt` at build time if one of the field `python_version` or `python_full_version` is specified in the `requires` section of your `Pipfile`.

 Specify a Buildpack Version
 ---------------------------
@@ -61,26 +62,32 @@ Specify a Python Runtime

 Supported runtime options include:

- `python-3.7.4`
- `python-3.6.9`
- `python-2.7.17`
+- `python-3.8.6`
+- `python-3.7.9`
+- `python-3.6.12`
+- `python-2.7.18`

 ## Tests

 The buildpack tests use [Docker](https://www.docker.com/) to simulate
 Heroku's [stack images.](https://devcenter.heroku.com/articles/stack)

-To run the test suite:
+To run the test suite against the default stack:

 ```
 make test
 ```

-Or to test in a particular stack:
+Or to test against a particular stack:

 ```
-make test-heroku-18
-make test-heroku-16
+make test STACK=heroku-16
+```
+
+To run only a subset of the tests:
+
+```
+make test TEST_CMD=tests/versions
 ```

 The tests are run via the vendored
@@ -16,7 +16,9 @@
 set -eo pipefail

 # Boostrap the Buildpack Standard Library.
-export BPLOG_PREFIX="buildpack.python"
+# Disable unused env var warning since shellcheck doesn't know about the stdlib.
+# shellcheck disable=2034
+BPLOG_PREFIX="buildpack.python"
 export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}

 [ "$BUILDPACK_XTRACE" ] && set -o xtrace
@@ -63,37 +65,13 @@ PYPY36="pypy3.6"

 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
-# If pip doesn't match this version (the version we install), run the installer.
-PIP_UPDATE="20.0.2"
-
-for file in "$BUILD_DIR/runtime.txt" "$CACHE_DIR/.heroku/python-version" ; do
-    [ -f "$file" ] || continue
-
-    version=$(tr -d '[:space:]' < "$file")
-
-    case "$version" in "$PY34"*)
-        # Python 3.4 support was dropped in pip >= 19.2.
-        PIP_UPDATE="19.1.1"
-        break
-        ;;
-    esac
-done
-
-if [[ -f "$BUILD_DIR/Pipfile" ]]; then
-    # Do not force pipenv users to re-install pipenv locally.
-    PIP_UPDATE="9.0.2"
-fi
-
-export DEFAULT_PYTHON_STACK PIP_UPDATE
-export PY37 PY36 PY35 PY27 PY34

 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.
 # The `warnings` subscript then greps through this for common problems and guides
 # the user towards resolution of known issues.
 WARNINGS_LOG=$(mktemp)
-export WARNINGS_LOG
-export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
+RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION

 # The buildpack ships with a few executable tools (e.g. pip-grep, etc).
 # This installs them into the path, so we can execute them directly.
@@ -159,6 +137,10 @@ export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PAT
 export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
 export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH

+# Global pip options (https://pip.pypa.io/en/stable/user_guide/#environment-variables).
+# Disable pip's warnings about EOL Python since we show our own.
+export PIP_NO_PYTHON_VERSION_WARNING=1
+
 # The Application Code
 # --------------------

@@ -175,7 +157,7 @@ cd "$BUILD_DIR"
 # - Once the build is complete, `~/.heroku/{known-paths}` is copied back into the cache.

 # Create the cache directory, if it doesn't exist.
-mkdir -p "$CACHE_DIR"
+mkdir -p "$CACHE_DIR/.heroku"

 # Restore old artifacts from the cache.
 mkdir -p .heroku
@@ -207,7 +189,7 @@ source "$BIN_DIR/steps/hooks/pre_compile"
 # continue to use that version of Python in perpituity (warnings will be raised if
 # they are out–of–date).
 if [ -f "$CACHE_DIR/.heroku/python-version" ]; then
-  DEFAULT_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
+  CACHED_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
 fi

 # We didn't always record the stack version. This code is in place because of that.
@@ -217,8 +199,6 @@ else
  CACHED_PYTHON_STACK=$STACK
 fi

-export CACHED_PYTHON_STACK
-
 # Pipenv Python version support.
 # Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version).
 # Convert it to a runtime.txt file.
@@ -226,9 +206,14 @@ export CACHED_PYTHON_STACK
 # shellcheck source=bin/steps/pipenv-python-version
 source "$BIN_DIR/steps/pipenv-python-version"

-# If no runtime was provided by the user, assume the default Python runtime version.
-if [ ! -f runtime.txt ]; then
-  echo "$DEFAULT_PYTHON_VERSION" > runtime.txt
+if [[ -f runtime.txt ]]; then
+  mcount "version.reason.python.specified"
+elif [[ -n "${CACHED_PYTHON_VERSION:-}" ]]; then
+  mcount "version.reason.python.cached"
+  echo "${CACHED_PYTHON_VERSION}" > runtime.txt
+else
+  mcount "version.reason.python.default"
+  echo "${DEFAULT_PYTHON_VERSION}" > runtime.txt
 fi

 # Create the directory for .profile.d, if it doesn't exist.
@@ -259,17 +244,8 @@ mtime "python.install.time" "${start}"
 # shellcheck source=bin/steps/pipenv
 source "$BIN_DIR/steps/pipenv"

-# Uninstall removed dependencies with Pip.
-# The buildpack will automatically remove any declared dependencies (in requirements.txt)
-# that were explicitly removed. This machinery is a bit complex, but it is not complicated.
-(( start=$(nowms) ))
-# shellcheck source=bin/steps/pip-uninstall
-source "$BIN_DIR/steps/pip-uninstall"
-mtime "pip.uninstall.time" "${start}"
-
 # If no requirements.txt file given, assume `setup.py develop` is intended.
 # This allows for people to ship a setup.py application to Heroku
-# (which is rare, but I vouch that it should work!)

 if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
  echo "-e ." > requirements.txt
@@ -1,21 +1,16 @@
 #!/usr/bin/env bash

-DEFAULT_PYTHON_VERSION="python-3.6.10"
-LATEST_38="python-3.8.2"
-LATEST_37="python-3.7.6"
-LATEST_36="python-3.6.10"
-LATEST_35="python-3.5.7"
-LATEST_34="python-3.4.10"
-LATEST_27="python-2.7.17"
-PYPY_36="pypy3.6-7.2.0"
-PYPY_27="pypy2.7-7.2.0"
+# Disable unused env var warning, since shellcheck doesn't take into account
+# that this file is sourced. We don't want to use export since it exposes
+# the env vars to subprocesses.
+# shellcheck disable=2034

-export DEFAULT_PYTHON_VERSION \
-    LATEST_38 \
-    LATEST_37 \
-    LATEST_36 \
-    LATEST_35 \
-    LATEST_34 \
-    LATEST_27 \
-    PYPY_36 \
-    PYPY_27
+DEFAULT_PYTHON_VERSION="python-3.6.12"
+LATEST_38="python-3.8.6"
+LATEST_37="python-3.7.9"
+LATEST_36="python-3.6.12"
+LATEST_35="python-3.5.10"
+LATEST_34="python-3.4.10"
+LATEST_27="python-2.7.18"
+LATEST_PYPY_36="pypy3.6-7.3.1"
+LATEST_PYPY_27="pypy2.7-7.3.1"
@@ -0,0 +1,21 @@
+# Python Buildpack Install Steps
+
+TODO: Add context on Python install steps, such as why symlinking vs copying
+
+## Installing Python packages using Pip
+
+### Convention: Use `python` process to invoke Pip
+
+We don't use this convention (yet) but this is an upcoming change being considered.
+
+This is a bigger concern on Windows than it is in Linux environments, but an
+emerging convention in the Python community is to invoke pip using:
+
+```
+python3 -m pip [options]
+```
+
+Invoking pip this way ensures correct location - python knows where these
+packages are stored because it put them there (defaults to Python's pathing info).
+
+All normal command line options are available using this method.
@@ -22,6 +22,10 @@ source "$BIN_DIR/utils"
 if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then

  if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then
+
+    puts-warn "The vendored GDAL package in the Heroku Python Buildpack now deprecated."
+    puts-warn "To enable GDAL use an alternative buildpack is available here - https://github.com/heroku/heroku-geo-buildpack"
+
    echo "-----> Noticed GDAL. Bootstrapping gdal."
    mkdir -p .heroku/vendor
    # Download and extract cryptography into target vendor directory.
@@ -24,6 +24,9 @@ source "$BIN_DIR/utils"
 if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
  mcount "buildvar.BUILD_WITH_GEO_LIBRARIES"

+  puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."
+  puts-warn "An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
+
  if [ ! -f ".heroku/vendor/bin/proj" ]; then
    echo "-----> Bootstrapping gdal, geos, proj."
    mkdir -p .heroku/vendor
@@ -38,9 +38,16 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
        mcount "failure.none-version"
    fi

+    if grep -qi '^django==1.*' requirements.txt; then
+        puts-warn "Your Django version is nearing the end of its community support."
+        puts-warn "Upgrade to continue to receive security updates and for the best experience with Django."
+        puts-warn "For more information, check out https://www.djangoproject.com/download/#supported-versions"
+    fi
+
    if [ ! -f "$BUILD_DIR/.heroku/python/bin/pip" ]; then
        exit 1
    fi
+
    /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
    PIP_STATUS="${PIPESTATUS[0]}"
    set -e
@@ -52,12 +59,9 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
        exit 1
    fi

-    # Smart Requirements handling
    cp requirements.txt .heroku/python/requirements-declared.txt
    /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt

-    echo
-
    # Install test dependencies, for CI.
    if [ "$INSTALL_TEST" ]; then
        if [[ -f "$1/requirements-test.txt" ]]; then
@@ -9,10 +9,7 @@ set -e
 if [[ -f Pipfile.lock ]]; then
    if [[ -f .heroku/python/Pipfile.lock.sha256 ]]; then
        if [[ $(openssl dgst -sha256 Pipfile.lock) == $(cat .heroku/python/Pipfile.lock.sha256) ]]; then
-            # Measure that we're using Pipenv.
-            mcount "tool.pipenv"
-
-            # Don't skip installation of there are git deps.
+            # Don't skip installation if there are git deps.
            if ! grep -q 'git' Pipfile.lock; then
                echo "Skipping installation, as Pipfile.lock hasn't changed since last deploy." | indent

@@ -27,6 +27,9 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
                if [ "$PYTHON" = 3.7 ]; then
                    echo "$LATEST_37" > "$BUILD_DIR/runtime.txt"
                fi
+                if [ "$PYTHON" = 3.8 ]; then
+                    echo "$LATEST_38" > "$BUILD_DIR/runtime.txt"
+                fi
            fi


@@ -8,6 +8,7 @@ PYTHON_VERSION=$(cat runtime.txt)
 VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"

 SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
+SECURITY_UPDATE_PYPY="The PyPy project has released a security update! Please consider upgrading to"

 ONLY_SUPPORTED_2_VERSION="Only the latest version of Python 2 is supported on the platform. Please consider upgrading to"

@@ -51,11 +52,9 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
    fi
  fi
  if [[ "$PYTHON_VERSION" == $PY27* ]]; then
+    puts-warn "$PYTHON_2_EOL_UPDATE"
+    echo "       Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq"
    # security update note
-    if [[ "$(date "+%Y")" -gt "2019" ]]; then
-      puts-warn "$PYTHON_2_EOL_UPDATE"
-      echo "       Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq"
-    fi
    if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then
      puts-warn "$ONLY_SUPPORTED_2_VERSION" "$LATEST_27"
      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
@@ -63,15 +62,15 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
  fi
  if [[ "$PYTHON_VERSION" == $PYPY27* ]]; then
    # security update note
-    if [ "$PYTHON_VERSION" != "$PYPY_27" ]; then
-      puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_27}?"
+    if [ "$PYTHON_VERSION" != "$LATEST_PYPY_27" ]; then
+      puts-warn "$SECURITY_UPDATE_PYPY" "$LATEST_PYPY_27"
      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
    fi
  fi
  if [[ "$PYTHON_VERSION" == $PYPY36* ]]; then
    # security update note
-    if [ "$PYTHON_VERSION" != "$PYPY_36" ]; then
-      puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_36}?"
+    if [ "$PYTHON_VERSION" != "$LATEST_PYPY_36" ]; then
+      puts-warn "$SECURITY_UPDATE_PYPY" "$LATEST_PYPY_36"
      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
    fi
  fi
@@ -81,19 +80,13 @@ else
  exit 1
 fi

+mcount "version.python.${PYTHON_VERSION}"
+
 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
    puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache"
    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
 fi

-# need to clear the cache for first time installing SQLite3,
-# since the version is changing and could lead to runtime errors
-# with compiled extensions.
-if [ -d .heroku/python ] && [ ! -f .heroku/python-sqlite3-version ] && python_sqlite3_check "$PYTHON_VERSION"; then
-  puts-step "Need to update SQLite3, clearing cache"
-  rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
-fi
-
 if [ -f .heroku/python-version ]; then
  if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then
      puts-step "Found $(cat .heroku/python-version), removing"
@@ -103,6 +96,24 @@ if [ -f .heroku/python-version ]; then
  fi
 fi

+# Check if we should reinstall python dependencies
+if [[ ! -f "$CACHE_DIR/.heroku/requirements.txt" ]]; then
+  # IF there's no cached dependencies, update cached version of requirements.txt
+  # This should only run for new apps and first deploys after this update
+  cp -R "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt"
+else
+  # IF there IS a cached directory, check for differences with the new one
+  if ! diff "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt" &> /dev/null; then
+    puts-step "Requirements file has been changed, clearing cached dependencies"
+    # if there are any differences, clear the Python cache
+    # Installing Python over again does not take noticably more time
+    cp -R "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt"
+    rm -rf .heroku/python
+    unset SKIP_INSTALL
+  else
+    puts-step "No change in requirements detected, installing from cache"
+  fi
+fi

 if [ ! "$SKIP_INSTALL" ]; then
    puts-step "Installing $PYTHON_VERSION"
@@ -110,8 +121,6 @@ if [ ! "$SKIP_INSTALL" ]; then
    # Prepare destination directory.
    mkdir -p .heroku/python

-    mcount "version.python.$PYTHON_VERSION"
-
    if ! curl "${VENDORED_PYTHON}" -s | tar zxv -C .heroku/python &> /dev/null; then
      puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)."
      puts-warn "Aborting.  More info: https://devcenter.heroku.com/articles/python-support"
@@ -121,34 +130,58 @@ if [ ! "$SKIP_INSTALL" ]; then
  # Record for future reference.
  echo "$PYTHON_VERSION" > .heroku/python-version
  echo "$STACK" > .heroku/python-stack
-  FRESH_PYTHON=true

  hash -r
 fi

-# Heroku uses the get-pip utility maintained by the Python community to vendor Pip.
-# https://github.com/pypa/get-pip
-GETPIP_URL="https://lang-python.s3.amazonaws.com/etc/get-pip.py"
-GETPIP_PY="${TMPDIR:-/tmp}/get-pip.py"
-
-if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then
-    mcount "failure.python.get-pip"
-    echo "Failed to pull down get-pip"
-    exit 1
-fi
-
-# If Pip isn't up to date:
-if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
-
-  puts-step "Installing pip"
-
-  # Remove old installations.
-  rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
-  rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
-
-  /app/.heroku/python/bin/python "$GETPIP_PY" pip=="$PIP_UPDATE" &> /dev/null
-  /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
-fi
-
 set -e
+
+PIP_VERSION='20.1.1'
+# Must use setuptools <47.2.0 until we fix:
+# https://github.com/heroku/heroku-buildpack-python/issues/1006
+SETUPTOOLS_VERSION='47.1.1'
+WHEEL_VERSION='0.34.2'
+
+if [[ "${PYTHON_VERSION}" == ${PY34}* ]]; then
+  # Python 3.4 support was dropped in pip 19.2+, setuptools 44+ and wheel 0.34+.
+  PIP_VERSION='19.1.1'
+  SETUPTOOLS_VERSION='43.0.0'
+  WHEEL_VERSION='0.33.6'
+elif [[ "${PYTHON_VERSION}" == ${PY27}* || "${PYTHON_VERSION}" == ${PYPY27}* ]]; then
+  # Python 2.7 support was dropped in setuptools 45+.
+  SETUPTOOLS_VERSION='44.1.1'
+fi
+
+# We don't use get-pip.py, since:
+#  - it uses `--force-reinstall`, which is unnecessary here and slows down repeat builds
+#  - it means downloading pip twice (once embedded in get-pip.py, and again during
+#    the install, since get-pip.py can't install the embedded version directly)
+#  - we would still have to manage several versions of get-pip.py, to support older Pythons.
+# Instead, we use the pip wheel to install itself, using the method described here:
+# https://github.com/pypa/pip/issues/2351#issuecomment-69994524
+PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl"
+PIP_WHEEL_URL="https://lang-python.s3.amazonaws.com/common/${PIP_WHEEL_FILENAME}"
+PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}"
+
+if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then
+  mcount "failure.python.download-pip"
+  puts-warn "Failed to download pip"
+  exit 1
+fi
+
+if [[ -f "$BUILD_DIR/Pipfile" ]]; then
+  # The buildpack is pinned to old pipenv, which requires older pip.
+  # Pip 9.0.2 doesn't support installing itself from a wheel, so we have to use split
+  # versions here (ie: installer pip version different from target pip version).
+  PIP_VERSION='9.0.2'
+  PIP_TO_INSTALL="pip==${PIP_VERSION}"
+else
+  PIP_TO_INSTALL="${PIP_WHEEL}"
+fi
+
+puts-step "Installing pip ${PIP_VERSION}, setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}"
+
+/app/.heroku/python/bin/python "${PIP_WHEEL}/pip" install --quiet --disable-pip-version-check --no-cache \
+  "${PIP_TO_INSTALL}" "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}"
+
 hash -r
@@ -67,7 +67,6 @@ buildpack_sqlite3_install() {
  puts-step "Installing SQLite3"

  if sqlite3_install "$BUILD_DIR/.heroku/python" ; then
-    echo "Sqlite3 successfully installed."
    mcount "success.python.sqlite3"
  else
    echo "Sqlite3 failed to install."
@@ -1,4 +1,4 @@
-FROM heroku/heroku:18-build.v16
+FROM heroku/heroku:18-build

 WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Skip the build for heroku-16.
-if [[ $S3_PREFIX == "heroku-16" ]]; then
-    exit 0
-fi
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building libffi…"
-
-SOURCE_TARBALL='ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz'
-
-curl -L $SOURCE_TARBALL | tar x
-
-cd libffi-3.1
-./configure --prefix=$OUT_PREFIX --disable-static &&
-make
-make install
-
-# Cleanup
-cd ..
@@ -3,9 +3,9 @@

 OUT_PREFIX=$1

-# Skip the build for heroku-16.
-if [[ $S3_PREFIX == "heroku-16" ]]; then
-    exit 0
+if [[ $S3_PREFIX != "cedar-14" ]]; then
+    echo "libmemcached only needs to be built for cedar-14, since newer stacks include it in the base image"
+    exit 1
 fi

 # fail hard
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy2.7
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy3.6
@@ -1,27 +1,4 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/

-OUT_PREFIX=$1
-BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
-export BIN_DIR
-
-# shellcheck source=bin/utils
-source "$BIN_DIR/steps/sqlite3"
-
-sqlite3_version
-echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
-sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
-
-echo "Building Python…"
-SOURCE_TARBALL='https://python.org/ftp/python/2.7.17/Python-2.7.17.tgz'
-curl -L $SOURCE_TARBALL | tar xz
-mv Python-2.7.17 src
-cd src
-
-./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
-make
-make install
-
-# Remove unneeded test directories, similar to the official Docker Python images:
-# https://github.com/docker-library/python
-find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+source $(dirname $0)/python2
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python2
@@ -1,32 +1,4 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/

-OUT_PREFIX=$1
-BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
-export BIN_DIR
-
-# shellcheck source=bin/utils
-source "$BIN_DIR/steps/sqlite3"
-
-sqlite3_version
-echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
-sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
-
-echo "Building Python…"
-SOURCE_TARBALL='https://python.org/ftp/python/3.7.2/Python-3.7.2.tgz'
-curl -L $SOURCE_TARBALL | tar xz
-mv Python-3.7.2 src
-cd src
-
-./configure --prefix=$OUT_PREFIX --with-ensurepip=no
-make
-make install
-
-# Remove unneeded test directories, similar to the official Docker Python images:
-# https://github.com/docker-library/python
-find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
-
-# Remove spare /
-LOCATION=${OUT_PREFIX%?}
-
-ln $LOCATION/bin/python3 $LOCATION/bin/python
+source $(dirname $0)/python3
@@ -1,32 +1,4 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/

-OUT_PREFIX=$1
-BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
-export BIN_DIR
-
-# shellcheck source=bin/utils
-source "$BIN_DIR/steps/sqlite3"
-
-sqlite3_version
-echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
-sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
-
-echo "Building Python…"
-SOURCE_TARBALL='https://python.org/ftp/python/3.4.9/Python-3.4.9.tgz'
-curl -L $SOURCE_TARBALL | tar xz
-mv Python-3.4.9 src
-cd src
-
-./configure --prefix=$OUT_PREFIX --with-ensurepip=no
-make
-make install
-
-# Remove unneeded test directories, similar to the official Docker Python images:
-# https://github.com/docker-library/python
-find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
-
-# Remove spare /
-LOCATION=${OUT_PREFIX%?}
-
-ln $LOCATION/bin/python3 $LOCATION/bin/python
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-sudo apt-get -qq update
-sudo apt-get install software-properties-common
-curl --fail --retry 3 --retry-delay 1 --connect-timeout 3 --max-time 30 https://cli-assets.heroku.com/install-ubuntu.sh | sh
@@ -14,9 +14,9 @@ case "$choice" in
  * ) exit 1;;
 esac

-originMaster=$(git rev-parse origin/master)
-echo "Tagging commit $originMaster with $newVersion... "
-git tag "$newVersion" "${originMaster:?}"
+originMain=$(git rev-parse origin/main)
+echo "Tagging commit $originMain with $newVersion... "
+git tag "$newVersion" "${originMain:?}"
 git push origin refs/tags/$newVersion

 heroku buildpacks:publish "$BP_NAME" "$newVersion"
@@ -31,7 +31,7 @@ if [ $(git tag | grep -q latest-version) ]; then
    echo "Updating latest-version tag"
    git tag -d latest-version
    git push origin :latest-version
-    git tag latest-version "${originMaster:?}"
+    git tag latest-version "${originMain:?}"
    git push --tags
 fi

@@ -1,5 +1,6 @@
 {
  "python": [
-    "heroku/python-getting-started"
+    "heroku/python-getting-started",
+    "sharpstone/python_default"
  ]
 }
@@ -1,3 +1,5 @@
 ---
 - - "./repos/python/python-getting-started"
-  - f56b90499ec11e1c9576da2f8c7331300e189db3
+  - main
+- - "./repos/python/python_default"
+  - ca947f69027b2a30be5d26f9a42f25e54f4d7a1a
@@ -1,3 +1,3 @@
 docopt==0.6.2
-bob-builder==0.0.17
-boto==2.48.0
+bob-builder==0.0.18
+boto==2.49.0
@@ -0,0 +1,33 @@
+require_relative '../spec_helper'
+
+describe "Heroku CI" do
+  it "works" do
+    before_deploy = Proc.new do
+      File.open("app.json", "w+") do |f|
+        f.puts <<~EOM
+          {
+            "environments": {
+              "test": {
+                "scripts": {
+                  "test": "nosetests"
+                }
+              }
+            }
+          }
+        EOM
+      end
+
+      run!("echo nose >> requirements.txt")
+    end
+
+    Hatchet::Runner.new("python_default", before_deploy: before_deploy).run_ci do |test_run|
+      expect(test_run.output).to match("Downloading nose")
+      expect(test_run.output).to match("OK")
+
+      test_run.run_again
+
+      expect(test_run.output).to match("installing from cache")
+      expect(test_run.output).to_not match("Downloading nose")
+    end
+  end
+end
@@ -1,10 +1,72 @@
 require_relative '../spec_helper'

-describe "Python!!!!!!!!!!!" do
-  it "🐍" do
-    Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app|
-      expect(app.output).to           match(/Installing pip/)
-      expect(app.run('python -V')).to match(/3.6.10/)
+describe "Python" do
+  describe "cache" do
+    it "functions correctly" do
+      Hatchet::Runner.new("python_default").deploy do |app|
+        expect(app.output).to match(/Installing pip/)
+
+        expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies")
+        expect(app.output).to_not match("No change in requirements detected, installing from cache")
+        expect(app.output).to_not match("No such file or directory")
+        expect(app.output).to_not match("cp: cannot create regular file")
+
+        # Redeploy with changed requirements file
+        run!(%Q{echo "" >> requirements.txt})
+        run!(%Q{echo "pygments" >> requirements.txt})
+        run!(%Q{git add . ; git commit --allow-empty -m next})
+        app.push!
+
+        # Check the cache to have cleared
+        expect(app.output).to match("Requirements file has been changed, clearing cached dependencies")
+        expect(app.output).to_not match("No dependencies found, preparing to install")
+        expect(app.output).to_not match("No change in requirements detected, installing from cache")
+
+        # With no changes on redeploy, the cache should be present
+        run!(%Q{git commit --allow-empty -m next})
+        app.push!
+
+        expect(app.output).to match("No change in requirements detected, installing from cache")
+        expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies")
+        expect(app.output).to_not match("No dependencies found, preparing to install")
+      end
+    end
+  end
+
+  describe "python versions" do
+    let(:stack) { ENV["HEROKU_TEST_STACK"] || DEFAULT_STACK }
+    it "works with 3.7.6" do
+      version = "3.7.6"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack).deploy do |app|
+        expect(app.run('python -V')).to match(version)
+      end
+    end
+
+    it "works with 3.8.2" do
+      version = "3.8.2"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack).deploy do |app|
+        expect(app.run('python -V')).to match(version)
+      end
+    end
+
+    it "fails with a bad version" do
+      version = "3.8.2.lol"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack, allow_failure: true).deploy do |app|
+        expect(app.output).to match("not available for this stack")
+      end
+    end
+  end
+
+  it "getting started app has no relative paths" do
+    buildpacks = [
+      :default,
+      "https://github.com/sharpstone/force_absolute_paths_buildpack"
+    ]
+    Hatchet::Runner.new("python-getting-started", buildpacks: buildpacks).deploy do |app|
+      # Deploy works
    end
  end
 end
@@ -1,4 +1,4 @@
-ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/' + ENV['TRAVIS_REPO_SLUG'] + '.git'
+ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/heroku/heroku-buildpack-python.git'

 require 'rspec/core'
 require 'rspec/retry'
@@ -15,9 +15,10 @@ RSpec.configure do |config|
  end
 end

-if ENV['TRAVIS']
-  # Don't execute tests against "merge" commits
-  exit 0 if ENV['TRAVIS_PULL_REQUEST'] != 'false' && ENV['TRAVIS_BRANCH'] == 'master'
-end
+DEFAULT_STACK = 'heroku-18'

-DEFAULT_STACK = 'heroku-16'
+def run!(cmd)
+  out = `#{cmd}`
+  raise "Error running command #{cmd} with output: #{out}" unless $?.success?
+  return out
+end
@@ -0,0 +1,4 @@
+set -euo pipefail
+
+echo "post_compile ran!"
+echo "post_compile env: $(printenv | cut -d '=' -f 1 | sort | xargs)."
@@ -0,0 +1,4 @@
+set -euo pipefail
+
+echo "pre_compile ran!"
+echo "pre_compile env: $(printenv | cut -d '=' -f 1 | sort | xargs)."
@@ -0,0 +1,2 @@
+pwned-passwords-django==1.4
+Django==2.1
@@ -0,0 +1 @@
+Django==1.11
@@ -1 +1 @@
-pypy2.7-7.2.0
+pypy2.7-7.3.1
@@ -0,0 +1 @@
+pypy2.7-7.2.0
@@ -1 +1 @@
-pypy3.6-7.2.0
+pypy3.6-7.3.1
@@ -0,0 +1 @@
+pypy3.6-7.2.0
@@ -1 +1 @@
-python-2.7.17
+python-2.7.18
@@ -1 +0,0 @@
-requests
@@ -1 +0,0 @@
-python-2.7.99
@@ -1 +0,0 @@
-flask
@@ -1 +0,0 @@
-python-3.4.99
@@ -1 +1 @@
-python-3.5.7
+python-3.5.10
@@ -1 +0,0 @@
-flask
@@ -1 +0,0 @@
-python-3.5.99
@@ -1 +0,0 @@
-flask
@@ -1 +0,0 @@
-python-3.6.99
@@ -1 +0,0 @@
-flask
@@ -1 +0,0 @@
-python-3.7.99
@@ -1 +0,0 @@
-flask
@@ -5,8 +5,9 @@
 source "bin/default_pythons"

 testAirflow() {
-  export SLUGIFY_USES_TEXT_UNIDECODE="yes"
-  compile "airflow"
+  local env_dir="$(mktmpdir)"
+  echo 'yes' > "${env_dir}/SLUGIFY_USES_TEXT_UNIDECODE"
+  compile 'airflow' '' "${env_dir}"
  assertCaptured "apache-airflow==1.10.2"
  assertCapturedSuccess
 }
@@ -17,19 +18,30 @@ testCollectstatic() {
 }

 testGEOS() {
-  export BUILD_WITH_GEO_LIBRARIES=1
-  compile "geos"
+  local env_dir="$(mktmpdir)"
+  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
+  compile 'geos' '' "${env_dir}"
  assertCaptured "geos"
  assertCapturedSuccess
 }

+testGEOSDeprecation() {
+  local env_dir="$(mktmpdir)"
+  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
+  compile 'geos' '' "${env_dir}"
+  assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated.
+ !     An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
+  assertCapturedSuccess
+}
+
 testNLTK() {
+  local env_dir="$(mktmpdir)"
  # NOTE: This is a RuntimeWarning emitted by Python 3's runpy.py script
  # which is what is used when you call `python -m <module>`. This is due to
  # how nltk imports things. It's not actually an error, but it would probably
  # be bad to silence in Production.
-  export PYTHONWARNINGS="ignore::RuntimeWarning"
-  compile "nltk"
+  echo 'ignore::RuntimeWarning' > "${env_dir}/PYTHONWARNINGS"
+  compile 'nltk' '' "${env_dir}"
  assertCaptured "[nltk_data] Downloading package city_database" "STD_ERR"
  assertCapturedSuccess
 }
@@ -48,7 +60,6 @@ testPysqlite() {

 testSqliteInstall() {
  compile "pythonDefault"
-  assertCaptured "Sqlite3 successfully installed."
  assertNotCaptured "Sqlite3 failed to install."
  assertCapturedSuccess
 }
@@ -44,6 +44,7 @@ testStandardRequirements() {

 testPipenv() {
  compile "pipenv"
+  assertCaptured "Installing pip 9.0.2, setuptools 47.1.1 and wheel 0.34.2"
  assertCapturedSuccess
 }

@@ -74,6 +75,62 @@ testNoRequirements() {
  assertCapturedError
 }

+testWarnOldDjango() {
+  compile "old-django"
+  assertCaptured "Your Django version is nearing the end of its community support."
+  assertCapturedSuccess
+}
+
+testDontWarnOldDjango() {
+  compile "not-old-django"
+  assertNotCaptured "Your Django version is nearing the end of its community support."
+  assertCapturedSuccess
+}
+
+testHooks() {
+  # Test that the hooks are called correctly, and that the environment contains
+  # the app's config vars but no unexpected env vars from the buildpack.
+  local env_dir="$(mktmpdir)"
+  echo 'test' > "${env_dir}/SOME_APP_CONFIG_VAR"
+  local expected_env_vars=(
+    _
+    BIN_DIR
+    BUILD_DIR
+    BUILDPACK_LOG_FILE
+    CACHE_DIR
+    C_INCLUDE_PATH
+    CPLUS_INCLUDE_PATH
+    ENV_DIR
+    EXPORT_PATH
+    HOME
+    LANG
+    LD_LIBRARY_PATH
+    LIBRARY_PATH
+    OLDPWD
+    PATH
+    PIP_NO_PYTHON_VERSION_WARNING
+    PKG_CONFIG_PATH
+    PROFILE_PATH
+    PWD
+    PYTHONUNBUFFERED
+    SHLVL
+    SOME_APP_CONFIG_VAR
+    STACK
+    VENDOR_URL
+  )
+  if [[ "${STACK}" == "cedar-14" || "${STACK}" == "heroku-16" ]]; then
+    # Remove "OLDPWD" from expected_env_vars since for bash <4.4 it's not exported to subshells:
+    # https://github.com/heroku/heroku-buildpack-python/pull/1011#issuecomment-665117835
+    read -ra expected_env_vars <<< "${expected_env_vars[@]/OLDPWD/}"
+  fi
+  compile 'hooks' '' "${env_dir}"
+  assertCaptured "pre_compile ran!"
+  assertCaptured "pre_compile env: ${expected_env_vars[*]}."
+  assertCaptured "post_compile ran!"
+  assertCaptured "post_compile env: ${expected_env_vars[*]}."
+  assertCapturedSuccess
+}
+
 pushd $(dirname 0) >/dev/null
 popd >/dev/null

@@ -9,6 +9,7 @@ testPythonDefault() {
  compile "pythonDefault"
  assertCaptured $DEFAULT_PYTHON_VERSION
  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
  assertCaptured "Installing SQLite3"
  assertCapturedSuccess
 }
@@ -18,12 +19,9 @@ testPython2() {
    echo $LATEST_27 > "runtime.txt"
    compile "python2"
    assertCaptured $LATEST_27
-    if [[ $(date "+%Y") > "2019" ]]; then
-      assertCaptured "python-2-7-eol-faq";
-    else
-      assertNotCaptured "python-2-7-eol-faq";
-    fi
+    assertCaptured "python-2-7-eol-faq";
    assertNotCaptured "security update"
+    assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
    assertCaptured "Installing SQLite3"
    assertCapturedSuccess
 }
@@ -31,61 +29,32 @@ testPython2() {
 testPython2_warn() {
    compile "python2_warn"
    assertCaptured "python-2.7.15"
-    if [[ $(date "+%Y") > "2019" ]]; then
-      assertCaptured "python-2-7-eol-faq";
-    else
-      assertNotCaptured "python-2-7-eol-faq";
-    fi
+    assertCaptured "python-2-7-eol-faq";
    assertCaptured "Only the latest version"
    assertCaptured "Installing SQLite3"
    assertCapturedSuccess
 }

-testPython2_fail() {
-    compile "python2_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
-}
-
 testPython3_4() {
  compile "python3_4"
  assertCaptured $LATEST_34
  assertNotCaptured "security update"
-  # if cedar 14 and legacy binaries, fail. if cedar 14 and staging, succeed.
-  if [[ ! -n $USE_STAGING_BINARIES ]] && [[ $STACK == "cedar-14" ]]; then
-      assertCapturedError
-# if heroku 18 and legacy binaries, succeed. if heroku 18 and staging, fail.
-  elif [[ -n $USE_STAGING_BINARIES ]] && [[ $STACK == "heroku-18" ]]; then
-      assertCapturedError
-  else
-#   all else succeed
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing pip 19.1.1, setuptools 43.0.0 and wheel 0.33.6"
+  assertCapturedSuccess
 }

 testPython3_4_warn() {
  compile "python3_4_warn"
  assertCaptured "python-3.4.9"
  assertCaptured "security update!"
-# if heroku 18 and legacy binaries, succeed. if heroku 18 and staging, fail.
-  if [[ -n $USE_STAGING_BINARIES ]] && [[ $STACK == "heroku-18" ]]; then
-      assertCapturedError
-  else
-#   all else succeed
-    assertCapturedSuccess
-  fi
-}
-
-testPython3_4_fail() {
-    compile "python3_4_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
+  assertCapturedSuccess
 }

 testPython3_5() {
  compile "python3_5"
  assertCaptured $LATEST_35
  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
  assertCaptured "Installing SQLite3"
  assertCapturedSuccess
 }
@@ -97,17 +66,12 @@ testPython3_5_warn() {
  assertCapturedSuccess
 }

-testPython3_5_fail() {
-    compile "python3_5_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
-}
-
 testPython3_6() {
  updateVersion "python3_6" $LATEST_36
  compile "python3_6"
  assertCaptured $LATEST_36
  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
  assertCaptured "Installing SQLite3"
  assertCapturedSuccess
 }
@@ -120,12 +84,6 @@ testPython3_6_warn() {
  assertCapturedSuccess
 }

-testPython3_6_fail() {
-    compile "python3_6_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
-}
-
 testPython3_7() {
  updateVersion "python3_7" $LATEST_37
  compile "python3_7"
@@ -134,6 +92,7 @@ testPython3_7() {
  else
    assertNotCaptured "security update"
    assertCaptured $LATEST_37
+    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
    assertCaptured "Installing SQLite3"
    assertCapturedSuccess
  fi
@@ -151,13 +110,21 @@ testPython3_7_warn() {
  fi
 }

-testPython3_7_fail() {
-  compile "python3_7_fail"
-  assertCaptured "Aborting"
-  assertCapturedError
+testPython3_8() {
+  updateVersion "python3_8" $LATEST_38
+  compile "python3_8"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertNotCaptured "security update"
+    assertCaptured $LATEST_38
+    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+    assertCaptured "Installing SQLite3"
+    assertCapturedSuccess
+  fi
 }

-testPython3_7_warn() {
+testPython3_8_warn() {
  compile "python3_8_warn"
  if [[ $STACK = "cedar-14" ]]; then
    assertCapturedError
@@ -169,47 +136,79 @@ testPython3_7_warn() {
  fi
 }

-testPython3_8() {
-  updateVersion "python3_8" $LATEST_38
-  compile "python3_8"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertNotCaptured "security update"
-    assertCaptured $LATEST_38
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
-  fi
-}
-
-testPython3_8_fail() {
-  compile "python3_8_fail"
+testPythonVersionInvalid() {
+  compile "python_version_invalid"
+  assertCaptured "Requested runtime (python-3.8.99) is not available for this stack"
  assertCaptured "Aborting"
  assertCapturedError
 }

 testPypy3_6() {
  compile "pypy3_6"
+  assertCaptured "Installing pypy"
+  assertNotCaptured "security update"
+  assertCaptured "$LATEST_PYPY_36"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCapturedSuccess
+}
+
+testPypy3_6_warn() {
+  compile "pypy3_6_warn"
  if [[ $STACK = "cedar-14" ]]; then
    assertCapturedError
  else
    assertCaptured "Installing pypy"
-    assertCaptured "$PYPY_36"
+    assertCaptured "security update!"
+    assertCaptured "$LATEST_PYPY_36"
    assertCapturedSuccess
  fi
 }

 testPypy2_7() {
  compile "pypy2_7"
+  assertCaptured "Installing pypy"
+  assertNotCaptured "security update"
+  assertCaptured "$LATEST_PYPY_27"
+  assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
+  assertCapturedSuccess
+}
+
+testPypy2_7_warn() {
+  compile "pypy2_7_warn"
  if [[ $STACK = "cedar-14" ]]; then
    assertCapturedError
  else
    assertCaptured "Installing pypy"
-    assertCaptured "$PYPY_27"
+    assertCaptured "security update!"
+    assertCaptured "$LATEST_PYPY_27"
    assertCapturedSuccess
  fi
 }

+testStickyPythonVersion() {
+  local cache_dir="$(mktmpdir)"
+  compile "python3_6_warn" "$cache_dir"
+  assertCaptured "Installing python-3.6.7"
+  assertCapturedSuccess
+  compile "no-runtime-txt" "$cache_dir"
+  assertCaptured "Installing python-3.6.7"
+  assertCapturedSuccess
+  # Whilst this file seems like an implementation detail (so something that should
+  # not be tested), we must guarantee the filename remains consistent for backwards
+  # compatibility across buildpack versions for already-built apps.
+  assertFile "python-3.6.7" ".heroku/python-version"
+}
+
+testPythonVersionChange() {
+  local cache_dir="$(mktmpdir)"
+  compile "python3_6_warn" "$cache_dir"
+  assertCaptured "Installing python-3.6.7"
+  assertCapturedSuccess
+  compile "python3_6" "$cache_dir"
+  assertCaptured "Found python-3.6.7, removing"
+  assertCapturedSuccess
+}
+
 pushd $(dirname 0) >/dev/null
 popd >/dev/null

@@ -56,21 +56,6 @@ resetCapture()
  unset rtrn # deprecated
 }

-detect()
-{
-  capture ${BUILDPACK_HOME}/bin/detect ${BUILD_DIR}
-}
-
-compile()
-{
-  capture ${BUILDPACK_HOME}/bin/compile ${BUILD_DIR} ${CACHE_DIR}
-}
-
-release()
-{
-  capture ${BUILDPACK_HOME}/bin/release ${BUILD_DIR}
-}
-
 updateVersion()
 {
  echo "$2" > "test/fixtures/${1}/runtime.txt"
@@ -234,13 +219,18 @@ default_process_types_cleanup() {
  fi
 }

+run_in_clean_env() {
+  # Prevent stray environment variables from outside the test runner being exposed to tests.
+  env -i HOME="${HOME}" LANG="${LANG}" PATH="${PATH}" STACK="${STACK}" "$@"
+}
+
 compile() {
  default_process_types_cleanup
  bp_dir=$(mktmpdir)
  compile_dir=$(mktmpdir)
  cp -a $(pwd)/* ${bp_dir}
  cp -a ${bp_dir}/test/fixtures/$1/. ${compile_dir}
-  capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3
+  capture run_in_clean_env ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3
 }

 compileDir() {
@@ -252,13 +242,13 @@ compileDir() {
  local env_dir=$3

  cp -a $(pwd)/* ${bp_dir}
-  capture ${bp_dir}/bin/compile ${compile_dir} ${cache_dir} ${env_dir}
+  capture run_in_clean_env ${bp_dir}/bin/compile ${compile_dir} ${cache_dir} ${env_dir}
 }

 release() {
  bp_dir=$(mktmpdir)
  cp -a $(pwd)/* ${bp_dir}
-  capture ${bp_dir}/bin/release ${bp_dir}/test/fixtures/$1
+  capture run_in_clean_env ${bp_dir}/bin/release ${bp_dir}/test/fixtures/$1
 }

 assertFile() {
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-if [[ ! "$STACK" ]]; then
-    echo '$STACK must be set! (heroku-16 | cedar-14)'
-    exit 1
-fi
-
-if [[ "$STACK" == "cedar-14" ]]; then
-    make test-cedar-14
-    exit $?
-fi
-
-if [[ "$STACK" == "heroku-16" ]]; then
-    make test-heroku-16
-    exit $?
-fi
-
-if [[ "$STACK" == "heroku-18" ]]; then
-    make test-heroku-18
-    exit $?
-fi
@@ -1,204 +0,0 @@
-#!/bin/sh
-
-# taken from
-# https://github.com/ryanbrainard/heroku-buildpack-testrunner/blob/master/lib/test_utils.sh
-
-oneTimeSetUp()
-{
-   TEST_SUITE_CACHE="$(mktemp -d ${SHUNIT_TMPDIR}/test_suite_cache.XXXX)"
-}
-
-oneTimeTearDown()
-{
-  rm -rf ${TEST_SUITE_CACHE}
-}
-
-setUp()
-{
-  OUTPUT_DIR="$(mktemp -d ${SHUNIT_TMPDIR}/output.XXXX)"
-  STD_OUT="${OUTPUT_DIR}/stdout"
-  STD_ERR="${OUTPUT_DIR}/stderr"
-  BUILD_DIR="${OUTPUT_DIR}/build"
-  CACHE_DIR="${OUTPUT_DIR}/cache"
-  mkdir -p ${OUTPUT_DIR}
-  mkdir -p ${BUILD_DIR}
-  mkdir -p ${CACHE_DIR}
-}
-
-tearDown()
-{
-  rm -rf ${OUTPUT_DIR}
-}
-
-capture()
-{
-  resetCapture
-
-  LAST_COMMAND="$@"
-
-  "$@" >${STD_OUT} 2>${STD_ERR}
-  RETURN=$?
-  rtrn=${RETURN} # deprecated
-}
-
-resetCapture()
-{
-  if [ -f ${STD_OUT} ]; then
-    rm ${STD_OUT}
-  fi
-
-  if [ -f ${STD_ERR} ]; then
-    rm ${STD_ERR}
-  fi
-
-  unset LAST_COMMAND
-  unset RETURN
-  unset rtrn # deprecated
-}
-
-detect()
-{
-  capture ${BUILDPACK_HOME}/bin/detect ${BUILD_DIR}
-}
-
-compile()
-{
-  capture ${BUILDPACK_HOME}/bin/compile ${BUILD_DIR} ${CACHE_DIR}
-}
-
-release()
-{
-  capture ${BUILDPACK_HOME}/bin/release ${BUILD_DIR}
-}
-
-assertCapturedEquals()
-{
-  assertEquals "$@" "$(cat ${STD_OUT})"
-}
-
-assertCapturedNotEquals()
-{
-  assertNotEquals "$@" "$(cat ${STD_OUT})"
-}
-
-assertCaptured()
-{
-  assertFileContains "$@" "${STD_OUT}"
-}
-
-assertNotCaptured()
-{
-  assertFileNotContains "$@" "${STD_OUT}"
-}
-
-assertCapturedSuccess()
-{
-  assertEquals "Expected captured exit code to be 0; was <${RETURN}>" "0" "${RETURN}"
-  assertEquals "Expected STD_ERR to be empty; was <$(cat ${STD_ERR})>" "" "$(cat ${STD_ERR})"
-}
-
-# assertCapturedError [[expectedErrorCode] expectedErrorMsg]
-assertCapturedError()
-{
-  if [ $# -gt 1 ]; then
-    expectedErrorCode=${1}
-    shift
-  fi
-
-  expectedErrorMsg=${1:-""}
-
-  if [ -z ${expectedErrorCode} ]; then
-    assertTrue "Expected captured exit code to be greater than 0; was <${RETURN}>" "[ ${RETURN} -gt 0 ]"
-  else
-    assertTrue "Expected captured exit code to be <${expectedErrorCode}>; was <${RETURN}>" "[ ${RETURN} -eq ${expectedErrorCode} ]"
-  fi
-
-  assertFileContains "Expected STD_OUT to contain error <${expectedErrorMsg}>" "${expectedErrorMsg}" "${STD_OUT}"
-  assertEquals       "STD_ERR should always be empty" "" "$(cat ${STD_ERR})"
-}
-
-assertAppDetected()
-{
-  expectedAppType=${1?"Must provide app type"}
-
-  assertCapturedSuccess
-  assertEquals "${expectedAppType}" "$(cat ${STD_OUT})"
-}
-
-assertNoAppDetected()
-{
-  assertEquals "1" "${RETURN}"
-  assertEquals "no" "$(cat ${STD_OUT})"
-  assertEquals "" "$(cat ${STD_ERR})"
-}
-
-_assertContains()
-{
-  if [ 5 -eq $# ]; then
-    msg=$1
-    shift
-  elif [ ! 4 -eq $# ]; then
-    fail "Expected 4 or 5 parameters; Receieved $# parameters"
-  fi
-
-  needle=$1
-  haystack=$2
-  expectation=$3
-  haystack_type=$4
-
-  case "${haystack_type}" in
-    "file") grep -q -F -e "${needle}" ${haystack} ;;
-    "text") echo "${haystack}" | grep -q -F -e "${needle}" ;;
-  esac
-
-  if [ "${expectation}" != "$?" ]; then
-    case "${expectation}" in
-      0) default_msg="Expected <${haystack}> to contain <${needle}>" ;;
-      1) default_msg="Did not expect <${haystack}> to contain <${needle}>" ;;
-    esac
-
-    fail "${msg:-${default_msg}}"
-  fi
-}
-
-assertContains()
-{
-  _assertContains "$@" 0 "text"
-}
-
-assertNotContains()
-{
-  _assertContains "$@" 1 "text"
-}
-
-assertFileContains()
-{
-  _assertContains "$@" 0 "file"
-}
-
-assertFileNotContains()
-{
-  _assertContains "$@" 1 "file"
-}
-
-command_exists () {
-    type "$1" > /dev/null 2>&1 ;
-}
-
-assertFileMD5()
-{
-  expectedHash=$1
-  filename=$2
-
-  if command_exists "md5sum"; then
-    md5_cmd="md5sum ${filename}"
-    expected_md5_cmd_output="${expectedHash}  ${filename}"
-  elif command_exists "md5"; then
-    md5_cmd="md5 ${filename}"
-    expected_md5_cmd_output="MD5 (${filename}) = ${expectedHash}"
-  else
-    fail "no suitable MD5 hashing command found on this system"
-  fi
-
-  assertEquals "${expected_md5_cmd_output}" "`${md5_cmd}`"
-}