Release v183 (#1097 )

Bump heroku_hatchet from 7.3.0 to 7.3.1 (#1095 )
Bumps [heroku_hatchet](https://github.com/heroku/hatchet) from 7.3.0 to 7.3.1. - [Release notes](https://github.com/heroku/hatchet/releases) - [Changelog](https://github.com/heroku/hatchet/blob/main/CHANGELOG.md) - [Commits](https://github.com/heroku/hatchet/compare/v7.3.0...v7.3.1) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-05 23:10:16 +00:00 · 2020-10-12 12:31:44 +01:00 · 2020-10-12 10:01:20 +01:00 · 2020-10-07 19:44:33 +01:00 · 2020-10-07 15:10:20 +01:00 · 2020-10-06 10:25:07 +01:00
123 changed files with 1224 additions and 2135 deletions
@@ -0,0 +1,6 @@
+{
+  "productTag": "a1aB0000000MR0RIAW",
+  "issueTypeLabels": { "gus: story": "USER STORY", "gus: bug": "BUG P3" },
+  "defaultBuild": "Heroku Unscheduled",
+  "statusWhenClosed": "CLOSED"
+}
@@ -1 +1 @@
-* @heroku/languages
+* @heroku/languages
@@ -0,0 +1,15 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "c: dependencies"
+
+  - package-ecosystem: "bundler"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "c: dependencies"
@@ -1,12 +1,17 @@
 name: Check Changelog

-on: [pull_request]
+on:
+  pull_request:
+    types: [opened, reopened, edited, synchronize]

 jobs:
- build:
-   runs-on: ubuntu-latest
-   steps:
-   - uses: actions/checkout@v1
-   - name: Check that CHANGELOG is touched
-     run: |
-       cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
+  check:
+    runs-on: ubuntu-latest
+    if: |
+      !contains(github.event.pull_request.body, '[skip changelog]') &&
+      !contains(github.event.pull_request.body, '[changelog skip]') &&
+      !contains(github.event.pull_request.body, '[skip ci]')
+    steps:
+      - uses: actions/checkout@v1
+      - name: Check that CHANGELOG is touched
+        run: git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
@@ -10,3 +10,5 @@ buildpack/*

 builds/dockerenv.staging*
 builds/dockerenv.production
+
+test/scratch
@@ -1,7 +0,0 @@
-repos:
-   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
-    hooks:
-    -   id: check-yaml
-    -   id: end-of-file-fixer
-    -   id: trailing-whitespace
@@ -1,39 +1,45 @@
-language: ruby
-dist: trusty
-sudo: required
-
-rvm:
- 2.4.4
-before_script:
-  - gem install bundler -v 1.16.2
-  - bundle exec hatchet ci:setup
+language: minimal
+dist: focal
+branches:
+  only:
+  - main

 script:
-  - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
-  - docker run --rm -e "STACK=cedar-14" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-cedar-14 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-  - docker run --rm -e "STACK=heroku-16" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-16 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-  - docker run --rm -e "STACK=heroku-18" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-18 bash $TESTFOLDER
+  - make test STACK="${STACK}" TEST_CMD="${TEST_CMD}"

 jobs:
  include:
-  - stage: Bash linting (shellcheck)
-    sudo: false
+  - name: Bash linting (shellcheck)
    script: make check
-  - stage: Hatchet Integration
-    if: branch = master
-    name: Run Hatchet
-    script: "bundle exec rspec"
+  - name: Hatchet integration tests
+    if: env(HEROKU_API_USER) IS present AND env(HEROKU_API_KEY) IS present
+    language: ruby
+    rvm:
+      - 2.7
+    before_script:
+      - bundle exec hatchet ci:setup
+    script:
+      - PARALLEL_SPLIT_TEST_PROCESSES=11 bundle exec parallel_split_test spec/hatchet/
+
 env:
-  matrix:
-    - TESTFOLDER=test/run-deps
-    - TESTFOLDER=test/run-versions
-    - TESTFOLDER=test/run-features
+  jobs:
+    - STACK=cedar-14 TEST_CMD=test/run-deps
+    - STACK=cedar-14 TEST_CMD=test/run-versions
+    - STACK=cedar-14 TEST_CMD=test/run-features
+
+    - STACK=heroku-16 TEST_CMD=test/run-deps
+    - STACK=heroku-16 TEST_CMD=test/run-versions
+    - STACK=heroku-16 TEST_CMD=test/run-features
+
+    - STACK=heroku-18 TEST_CMD=test/run-deps
+    - STACK=heroku-18 TEST_CMD=test/run-versions
+    - STACK=heroku-18 TEST_CMD=test/run-features
+
+    - STACK=heroku-20 TEST_CMD=test/run-deps
+    - STACK=heroku-20 TEST_CMD=test/run-versions
+    - STACK=heroku-20 TEST_CMD=test/run-features
  global:
    - HATCHET_RETRIES=3
    - IS_RUNNING_ON_CI=true
-    - HATCHET_APP_LIMIT=5
+    - HATCHET_APP_LIMIT=80
    - HATCHET_DEPLOY_STRATEGY=git
-    - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
-    - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=
@@ -1,35 +1,145 @@
 # Python Buildpack Changelog

-# Master
+## Unreleased
+
+
+## v183 (2020-10-12)
+
+- Add support for Heroku-20 (#968).
+
+## v182 (2020-10-06)
+
+- Python 3.9.0 is now available (CPython) (#1090).
+- Migrate from the `lang-python` S3 bucket to `heroku-buildpack-python` (#1089).
+- Remove `vendor/shunit2` (#1086).
+- Replace `BUILDPACK_VENDOR_URL` and `USE_STAGING_BINARIES` with `BUILDPACK_S3_BASE_URL` (#1085).
+
+## v181 (2020-09-29)
+
+- PyPy 2.7 and 3.6, version 7.3.2 are now available (Note: PyPy support is in beta) (#1081).
+
+## v180 (2020-09-24)
+
+- Python 3.8.6 is now available (CPython) (#1072).
+
+## v179 (2020-09-23)
+
+- Remove duplicate pipenv metric event (#1070).
+- Emit metrics for how the Python version was chosen for an app (#1069).
+- Emit Python version metric events for all builds, not just clean installs (#1066).
+
+## v178 (2020-09-07)
+
+- Python 3.5.10 is now available (CPython) (#1062).
+
+## v177 (2020-08-18)
+
+- Python 3.6.12 and 3.7.9 are now available (CPython) (#1054).
+- The default Python version for new apps is now 3.6.12 (previously 3.6.11) (#1054).
+
+## v176 (2020-08-12)
+
+- Rebuild the Python 3.4.10 archives with the correct version of Python (#1048).
+- Fix the security update version check message for apps using PyPy (#1040).
+- Remove `vendor/test-utils` (#1043).
+
+## v175 (2020-08-05)
+
+- Update pip from 20.0.2 to 20.1.1 for Python 2.7 and Python 3.5+ (#1030).
+- Update setuptools from 39.0.1 to: (#1024)
+  - 44.1.1 for Python 2.7
+  - 43.0.0 for Python 3.4
+  - 47.1.1 for Python 3.5+
+- Switch the `heroku-buildpack-python` repository default branch from `master` to `main` (#1029).
+
+## v174 (2020-07-30)
+
+- For repeat builds, also manage the installed versions of setuptools/wheel, rather than just that of pip (#1007).
+- Install an explicit version of wheel rather than the latest release at the time (#1007).
+- Output the installed version of pip, setuptools and wheel in the build log (#1007).
+- Errors installing pip/setuptools/wheel are now displayed in the build output and fail the build early (#1007).
+- Install pip using itself rather than `get-pip.py` (#1007).
+- Disable pip's version check + cache when installing pip/setuptools/wheel (#1007).
+- Install setuptools from PyPI rather than a vendored copy (#1007).
+- Reduce the number of environment variables exposed to `bin/{pre,post}_compile` and other subprocesses (#1011).
+
+## v173 (2020-07-21)
+
+- Python 3.8.5 is now available (CPython).
+
+## v172 (2020-07-17)
+
+- Python 3.8.4 is now available (CPython).
+
+## v171 (2020-07-07)
+
+- Python 3.6.11 and 3.7.8 are now available (CPython).
+
+## v170 (2020-05-19)
+
+- Python 2.7.18, 3.5.9, 3.7.7 and 3.8.3 are now available (CPython).
+- PyPy 2.7 and 3.6, version 7.3.1 are now available (Note: PyPy support is in beta).
+- Docs: Fix explanation of runtime.txt generation when using pipenv.
+- Bugfix: Correctly detect Python version when using a `python_version` of `3.8` in `Pipfile.lock`.
+
+## v169 (2020-04-22)
+
+- Add a Hatchet test for python 3.8.2
+- Set Code Owners to @heroku/langauges
+- Bugfix: Caching on subsequent redeploys
+- Update tests to support latest version of Python
+
+## v168 (2020-04-06)
+
+- Doc: Update Readme with version numbers
+- update Code Owners to include the Heroku Buildpack Maintainers team
+- Deprecation warning: `BUILD_WITH_GEO_LIBRARIES` is now deprecated. See warning for details.
+- Clean up build log output
+- Update Python versions in README to match docs
+- Django version detection fixed, link updated
+
+## v167 (2020-03-26)
+
+- Add failcase for cache busting
+- Bugfix: Clearing pip dependencies
+
+## v166 (2020-03-05)
+
+- Correct ftp to https in vendored file
+- Warn for Django 1.11 approaching EOL, provide link to roadmap
+
+## v165 (2020-02-27)
+
+- Python 3.8.2 now available.
+
+## v164 (2020-02-20)

 - Update requirements.txt builds to use Pip 20.0.2
 - Download get-pip.py to tmpdir instead of root dir

--------------------------------------------------------------------------------
-
-# 163 (2019-12-23)
+## v163 (2019-12-23)

 - New pythons released:
  Python 3.8.1, 3.7.6, 3.6.10 (CPython)
  Beta Release: Pypy 2.7 and 3.6, version 7.2.0

-# 162 (2019-12-06)
+## v162 (2019-12-06)

 - Bug fix: fragile sqlite3 install

-# 161 (2019-12-2)
+## v161 (2019-12-2)

 - Bug fix: Sqlite3 version bump

-# 160 (2019-10-23)
+## v160 (2019-10-23)

 - Bugfix: Pipenv no longer installs twice in CI

-# 159 (2019-10-22)
+## v159 (2019-10-22)

 - Python 2.7.17 now available on Heroku 18 and 16.

-# 158 (2019-10-21)
+## v158 (2019-10-21)

 - Python 3.7.5 and 3.8.0 now available on Heroku 18 and 16.
 - Add support for Python 3.8 branch
@@ -37,11 +147,11 @@
  - Test Improvements
 - Add support for staging binary testing

-# 157 (2019-09-18)
+## v157 (2019-09-18)

 - Typo fixes

-# 156 (2019-09-12)
+## v156 (2019-09-12)

 - Python 3.6.9 and 3.7.4 now available.

@@ -49,125 +159,125 @@
 - Build utility and documentation updates
 - Bump Hatchet tests to point at new default python version.

-# 155 (2019-08-22)
+## v155 (2019-08-22)

 add docs and make target for heroku-18 bob builds

-# 154 (2019-07-17)
+## v154 (2019-07-17)

 Fix python 3.5.7 formula actually building 3.7.2

-# 153 (2019-06-21)
+## v153 (2019-06-21)

 Hotfix for broken heroku-16 deploys

-# 152 (2019-04-04)
+## v152 (2019-04-04)

 Python 3.7.3 now available.

-# 151 (2019-03-21)
+## v151 (2019-03-21)

 Python 3.5.7 and 3.4.10 now available on all Heroku stacks.

-# 150 (2019-03-13)
+## v150 (2019-03-13)

 Python 2.7.16 now available on all Heroku stacks.

-# 149 (2019-03-04)
+## v149 (2019-03-04)

 Hotfix for broken Cedar 14 deploys

-# 148 (2019-02-21)
+## v148 (2019-02-21)

 No user facing changes, improving internal metrics

-# 147 (2019-02-07)
+## v147 (2019-02-07)

 Python 3.7.2 and 3.6.8 now available on all Heroku stacks.

-# 146 (2018-11-11)
+## v146 (2018-11-11)

 Python 3.7.1, 3.6.7, 3.5.6 and 3.4.9 now available on all Heroku stacks.

-# 145 (2018-11-08)
+## v145 (2018-11-08)

 Testing and tooling expanded to better support new runtimes

-# 144 (2018-10-10)
+## v144 (2018-10-10)

 Switch to cautious upgrade for Pipenv install to ensure the pinned pip version
 is used with Pipenv

-# 143 (2018-10-09)
+## v143 (2018-10-09)

-Add support for detecting SLUGIFY_USES_TEXT_UNIDECODE, which is required to
+Add support for detecting `SLUGIFY_USES_TEXT_UNIDECODE`, which is required to
 install Apache Airflow version 1.10 or higher.

-# 142 (2018-10-08)
+## v142 (2018-10-08)

 Improvements to Python install messaging

-# 139, 140, 141
+## v139, 140, 141

 No user-facing changes, documenting for version clarity

-# 138 (2018-08-01)
+## v138 (2018-08-01)

 Use stack image SQLite3 instead of vendoring

-# 137 (2018-07-17)
+## v137 (2018-07-17)

 Prevent 3.7.0 from appearing as unsupported in buildpack messaging.

-# 136 (2018-06-28)
+## v136 (2018-06-28)

 Upgrade to 3.6.6 and support 3.7.0 on all runtimes.

-# 135 (2018-05-29)
+## v135 (2018-05-29)

 Upgrade Pipenv to v2018.5.18.

-# 134 (2018-05-02)
+## v134 (2018-05-02)

 Default to 3.6.5, bugfixes.

-# 133
+## v133

 Fixes for Pip 10 release.

-# 132
+## v132

 Improve pip installation, with the release of v9.0.2.

-# 131
+## v131

 Fix bug with pip.

-# 130
+## v130

 Better upgrade strategy for pip.

-# 129
+## v129

 Don't upgrade pip (from v128).

-# 128
+## v128

 Upgrade pip, pin to Pipenv v11.8.2.

-# 127
+## v127

 Pin to Pipenv v11.7.1.

-# 126
+## v126

 Bugfixes.

-# 125
+## v125

 Bugfixes.

-# 124
+## v124

 Update buildpack to automatically install `[dev-packages]` during Heroku CI Pipenv builds.

@@ -176,101 +286,101 @@ Update buildpack to automatically install `[dev-packages]` during Heroku CI Pipe
 - No longer warn if there is no `Procfile`.
 - Update Pipenv's "3.6" runtime specifier to point to "3.6.4".

-# 123
+## v123

 Update gunicorn `init.d` script to allow overrides.

-# 122
+## v122

 Update default Python to v3.6.4.

-# 121
+## v121

 Update default Python to v3.6.3.

-# 120
+## v120

 Use `$ pipenv --deploy`.

-# 119
+## v119

 Improvements to Pipenv support, warning on unsupported Python versions.

 - We now warn when a user is not using latest 2.x or 3.x Python.
 - Heroku now supports `[requires]` `python_full_version` in addition to `python_version`.

-# 118
+## v118

 Improvements to Pipenv support.

-# 117
+## v117

 Bug fix.

-# 116
+## v116

 Vendoring improvements.

 - Geos libraries should work on Heroku-16 now.
 - The libffi/libmemcached vendoring step is now skipped on Heroku-16 (since they are installed in the base image).

-# 115
+## v115

 Revert a pull request.

 - No longer using `sub_env` for `pip install` step.

-# 114
+## v114

 - Bugfixes.

 Blacklisting `PYTHONHOME` and `PYTHONPATH` for older apps. Upgrades to nltk support.

-# 113
+## v113

 Updates to Pipenv support.

-# 112
+## v112

 Bugfix.

 - Fixed grep output bug.

-# 111
+## v111

 Linting, bugfixes.

-# 110
+## v110

 Update default Python to 3.6.2.

-# 109
+## v109

 Update Default Python to 3.6.1, bugfixes.

 - Fixed automatic pip uninstall of dependencies removed from requirements.txt.

-# 108
+## v108

 Fix output for collectstatic step.

-# 107
+## v107

 Bugfix for C dependency installation.

-# 106
+## v106

- Don't install packages that could mess up packaging.
+Don't install packages that could mess up packaging.

 - The Python buildpack will automatically remove `six`, `pyparsing`, `appdirs`,
   `setuptools`, and `distribute` from a `requirements.txt` file now, as these
   packages are provided by the Python buildpack.

- # 105
+## v105

- Improvements to output messaging.
+Improvements to output messaging.

-# 104
+## v104

 General improvements.

@@ -280,52 +390,52 @@ General improvements.

 [395]: https://github.com/heroku/heroku-buildpack-python/pull/395

-## 103
+## v103

 Bug fixes and improvements.

 - Fix for Pipenv.
 - Fix for Heroku CI.
- Improve handling of WEB_CONCURRENCY when using multiple buildpacks.
- Adjust environment variables set during the build to more closely match those in the dyno environment (DYNO is now available, STACK is not).
+- Improve handling of `WEB_CONCURRENCY` when using multiple buildpacks.
+- Adjust environment variables set during the build to more closely match those in the dyno environment (`DYNO` is now available, `STACK` is not).
 - Restore the build cache prior to running bin/pre_compile.

-## 102
+## v102

 Buildpack code cleanup.

 - Improved messaging around NLTK.

-## 101
+## v101

 Updated setuptools installation method.

 - Improved pipenv support.

-## 100
+## v100

 Preliminary pipenv support.

-## 99
+## v99

 Cleanup.

-## 98
+## v98

 Official NLTK support and other improvements.

 - Support for `nltk.txt` file for declaring corpora to be downloaded.
- Leading zeros for auto-set WEB_CONCURRENCY.
+- Leading zeros for auto-set `WEB_CONCURRENCY`.

-## 97
+## v97

 Improved egg-link functionality.

-## 96
+## v96

 Bugfix.

-## 95
+## v95

 Improved output support.

@@ -361,7 +471,7 @@ Fixed bug with editable pip installations.

 Updated default Python 2.7.13.

- Python 2.7.13 uses UCS-4 build, more compatibile with linux wheels.
+- Python 2.7.13 uses UCS-4 build, more compatible with linux wheels.
 - Updated setuptools to v32.1.0.

 ## v86
@@ -387,7 +497,7 @@ Support for Heroku CI.

 ## v82 (2016-08-22)

-Update to library detection mechnisms (pip-pop).
+Update to library detection mechanisms (pip-pop).

 - Updated setuptools to v25.5.0

@@ -461,7 +571,7 @@ Revert to v66.

 ## v68 (2015-10-12)

-Fixed .heroku/venv error with modern apps.
+Fixed `.heroku/venv` error with modern apps.

 ## v67 (2015-10-12)

@@ -486,7 +596,6 @@ Updated Pip and Setuptools.
 - Setuptools updated to v18.3.2
 - Pip updated to v7.1.2

-
 ## v62 (2015-08-07)

 Updated Pip and Setuptools.
@@ -4,3 +4,5 @@ gem "rspec"
 gem "heroku_hatchet"
 gem "rspec-retry"
 gem "rake"
+gem "parallel_split_test"
+
@@ -1,71 +1,60 @@
 GEM
  remote: https://rubygems.org/
  specs:
-    activesupport (5.2.1)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-    concurrent-ruby (1.1.3)
-    diff-lcs (1.3)
+    diff-lcs (1.4.4)
    erubis (2.7.0)
-    excon (0.62.0)
-    heroics (0.0.25)
+    excon (0.76.0)
+    heroics (0.1.1)
      erubis (~> 2.0)
      excon
      moneta
      multi_json (>= 1.9.2)
-    heroku_hatchet (4.0.6)
+    heroku_hatchet (7.3.1)
      excon (~> 0)
-      minitest-retry (~> 0.1.9)
-      platform-api (~> 2)
-      repl_runner (~> 0.0.3)
+      platform-api (~> 3)
      rrrretry (~> 1)
-      thor (~> 0)
+      thor (~> 1)
      threaded (~> 0)
-    i18n (1.1.1)
-      concurrent-ruby (~> 1.0)
-    minitest (5.11.3)
-    minitest-retry (0.1.9)
-      minitest (>= 5.0)
    moneta (1.0.0)
-    multi_json (1.13.1)
-    platform-api (2.2.0)
-      heroics (~> 0.0.25)
+    multi_json (1.15.0)
+    parallel (1.19.2)
+    parallel_split_test (0.8.0)
+      parallel (>= 0.5.13)
+      rspec (>= 3.1.0)
+    platform-api (3.0.0)
+      heroics (~> 0.1.1)
      moneta (~> 1.0.0)
-    rake (12.3.1)
-    repl_runner (0.0.3)
-      activesupport
+      rate_throttle_client (~> 0.1.0)
+    rake (13.0.1)
+    rate_throttle_client (0.1.2)
    rrrretry (1.0.0)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-retry (0.6.1)
+      rspec-support (~> 3.9.0)
+    rspec-retry (0.6.2)
      rspec-core (> 3.3)
-    rspec-support (3.8.0)
-    thor (0.20.3)
-    thread_safe (0.3.6)
+    rspec-support (3.9.3)
+    thor (1.0.1)
    threaded (0.0.4)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)

 PLATFORMS
  ruby

 DEPENDENCIES
  heroku_hatchet
+  parallel_split_test
  rake
  rspec
  rspec-retry

 BUNDLED WITH
-   1.16.3
+   2.1.4
@@ -1,55 +1,63 @@
 # These targets are not files
-.PHONY: tests
+.PHONY: check test builder-image buildenv deploy-runtimes tools

-test: test-heroku-18 test-heroku-16 test-cedar-14
+STACK ?= heroku-18
+STACKS ?= cedar-14 heroku-16 heroku-18
+TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
+ENV_FILE ?= builds/dockerenv.default
+BUILDER_IMAGE_PREFIX := heroku-python-build
+
+ifeq ($(STACK),cedar-14)
+	# Cedar-14 doesn't have a build image varient.
+	STACK_IMAGE_TAG := heroku/cedar:14
+else
+	# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
+	STACK_IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
+endif

 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
 	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
 	@shellcheck -x bin/steps/hooks/*

-test-cedar-14:
-	@echo "Running tests in docker (cedar-14)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
-	@echo ""
-
-test-heroku-16:
-	@echo "Running tests in docker (heroku-16)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
-	@echo ""
-
-test-heroku-18:
-	@echo "Running tests in docker (heroku-18)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
-	@echo ""
-
-buildenv-heroku-16:
-	@echo "Creating build environment (heroku-16)..."
+test:
+	@echo "Running tests using: STACK=$(STACK) TEST_CMD='$(TEST_CMD)'"
 	@echo
-	@docker build --pull -f $(shell pwd)/builds/heroku-16.Dockerfile -t python-buildenv-heroku-16 .
+	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(STACK_IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
+	@echo
+
+builder-image:
+	@echo "Generating binary builder image for $(STACK)..."
+	@echo
+	@docker build --pull -f builds/$(STACK).Dockerfile -t "$(BUILDER_IMAGE_PREFIX)-$(STACK)" .
+	@echo
+
+buildenv: builder-image
+	@echo "Starting build environment for $(STACK)..."
 	@echo
 	@echo "Usage..."
 	@echo
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ bob build runtimes/python-2.7.13"
-	@echo "  $$ bob deploy runtimes/python-2.7.13"
+	@echo "  $$ bob build runtimes/python-X.Y.Z"
 	@echo
-	@docker run -it --rm python-buildenv-heroku-16
+	@docker run --rm -it --env-file="$(ENV_FILE)" -v $(PWD)/builds:/app/builds "$(BUILDER_IMAGE_PREFIX)-$(STACK)" bash

-buildenv-heroku-18:
-	@echo "Creating build environment (heroku-18)..."
+deploy-runtimes:
+ifndef RUNTIMES
+	$(error No runtimes specified! Use: "make deploy-runtimes RUNTIMES='python-X.Y.Z ...' [STACKS='heroku-18 ...'] [ENV_FILE=...]")
+endif
+	@echo "Using: RUNTIMES='$(RUNTIMES)' STACKS='$(STACKS)' ENV_FILE='$(ENV_FILE)'"
 	@echo
-	@docker build --pull -f $(shell pwd)/builds/heroku-18.Dockerfile -t python-buildenv-heroku-18 .
-	@echo
-	@echo "Usage..."
-	@echo
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ bob build runtimes/python-2.7.13"
-	@echo "  $$ bob deploy runtimes/python-2.7.13"
-	@echo
-	@docker run -it --rm python-buildenv-heroku-18
+	@set -eu; for stack in $(STACKS); do \
+		$(MAKE) builder-image STACK=$${stack}; \
+		for runtime in $(RUNTIMES); do \
+			echo "Generating/deploying $${runtime} for $${stack}..."; \
+			echo; \
+			docker run --rm -it --env-file="$(ENV_FILE)" "$(BUILDER_IMAGE_PREFIX)-$${stack}" bob deploy "runtimes/$${runtime}"; \
+			echo; \
+		done; \
+	done

 tools:
 	git clone https://github.com/kennethreitz/pip-pop.git
 	mv pip-pop/bin/* vendor/pip-pop/
-	rm -fr pip-pop
+	rm -rf pip-pop
@@ -86,31 +86,6 @@ Each version is given a distinguishing version number. If the Library as you rec

 If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library.

-get-pip.py license
------------------
-
-Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-
 pip-pop license
 ---------------

@@ -134,4 +109,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+THE SOFTWARE.
@@ -1,9 +0,0 @@
-[[source]]
-url = "https://pypi.python.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[dev-packages]
-
-[packages]
-"bob-builder" = "==0.0.13"
@@ -1,52 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "36d17c46a8b1b844b3cae475f6f42d6c0a9e59b2a9685cbcdc0985656a7a129f"
-        },
-        "host-environment-markers": {
-            "implementation_name": "cpython",
-            "implementation_version": "3.6.3",
-            "os_name": "posix",
-            "platform_machine": "x86_64",
-            "platform_python_implementation": "CPython",
-            "platform_release": "16.7.0",
-            "platform_system": "Darwin",
-            "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64",
-            "python_full_version": "3.6.3",
-            "python_version": "3.6",
-            "sys_platform": "darwin"
-        },
-        "pipfile-spec": 6,
-        "requires": {},
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.python.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "bob-builder": {
-            "hashes": [
-                "sha256:b4de49a8e436fcaf82236ea43f78413b4a4c92100726e382ab57b6bdfb38fe64",
-                "sha256:288e3e765c4890fe9a63ae52ac6b4a963c13fe508482c70ff701a5ae21b9a673"
-            ],
-            "version": "==0.0.13"
-        },
-        "boto": {
-            "hashes": [
-                "sha256:13be844158d1bd80a94c972c806ec8381b9ea72035aa06123c5db6bc6a6f3ead",
-                "sha256:deb8925b734b109679e3de65856018996338758f4b916ff4fe7bb62b6d7000d1"
-            ],
-            "version": "==2.48.0"
-        },
-        "docopt": {
-            "hashes": [
-                "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
-            ],
-            "version": "==0.6.2"
-        }
-    },
-    "develop": {}
-}
@@ -2,7 +2,7 @@

 # Heroku Buildpack: Python

-[![Build Status](https://travis-ci.org/heroku/heroku-buildpack-python.svg?branch=master)](https://travis-ci.org/heroku/heroku-buildpack-python)
+[![Build Status](https://travis-ci.com/heroku/heroku-buildpack-python.svg?branch=main)](https://travis-ci.com/heroku/heroku-buildpack-python)

 This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/buildpacks) for Python apps.

@@ -16,7 +16,7 @@ See it in Action
 $ ls
 my-application		requirements.txt	runtime.txt

-$ git push heroku master
+$ git push heroku main
 Counting objects: 4, done.
 Delta compression using up to 8 threads.
 Compressing objects: 100% (2/2), done.
@@ -26,7 +26,7 @@ remote: Compressing source files... done.
 remote: Building source:
 remote:
 remote: -----> Python app detected
-remote: -----> Installing python-3.7.4
+remote: -----> Installing python
 remote: -----> Installing pip
 remote: -----> Installing SQLite3
 remote: -----> Installing requirements with pip
@@ -44,9 +44,10 @@ A `requirements.txt` must be present at the root of your application's repositor

 To specify your python version, you also need a `runtime.txt` file - unless you are using the default Python runtime version.

-Current default Python Runtime: Python 3.6.9
+Current default Python Runtime: Python 3.6.12

-Alternatively, you can provide a `setup.py` file, or a `Pipfile`. Using `Pipenv` will generate `runtime.txt` based on `python-version` at build time.
+Alternatively, you can provide a `setup.py` file, or a `Pipfile`.
+Using `pipenv` will generate `runtime.txt` at build time if one of the field `python_version` or `python_full_version` is specified in the `requires` section of your `Pipfile`.

 Specify a Buildpack Version
 ---------------------------
@@ -61,26 +62,33 @@ Specify a Python Runtime

 Supported runtime options include:

- `python-3.7.4`
- `python-3.6.9`
- `python-2.7.17`
+- `python-3.9.0`
+- `python-3.8.6`
+- `python-3.7.9`
+- `python-3.6.12`
+- `python-2.7.18`

 ## Tests

 The buildpack tests use [Docker](https://www.docker.com/) to simulate
 Heroku's [stack images.](https://devcenter.heroku.com/articles/stack)

-To run the test suite:
+To run the test suite against the default stack:

 ```
 make test
 ```

-Or to test in a particular stack:
+Or to test against a particular stack:

 ```
-make test-heroku-18
-make test-heroku-16
+make test STACK=heroku-16
+```
+
+To run only a subset of the tests:
+
+```
+make test TEST_CMD=tests/versions
 ```

 The tests are run via the vendored
@@ -16,7 +16,9 @@
 set -eo pipefail

 # Boostrap the Buildpack Standard Library.
-export BPLOG_PREFIX="buildpack.python"
+# Disable unused env var warning since shellcheck doesn't know about the stdlib.
+# shellcheck disable=2034
+BPLOG_PREFIX="buildpack.python"
 export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}

 [ "$BUILDPACK_XTRACE" ] && set -o xtrace
@@ -35,23 +37,27 @@ ENV_DIR=$3
 # Export Path variables, for use in sub-scripts.
 export BUILD_DIR CACHE_DIR ENV_DIR

-# Set the Buildpack's internet target for downloading Python distributions.
-# The user can provide BUILDPACK_VENDOR_URL to specify a custom target.
+# Set the base URL for downloading buildpack assets like Python runtimes.
+# The user can provide BUILDPACK_S3_BASE_URL to specify a custom target.
 # Note: this is designed for non-Heroku use, as it does not use the user-provided
 # environment variable mechanism (the ENV_DIR).
-VENDOR_URL="https://lang-python.s3.amazonaws.com/$STACK"
-if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then
-    VENDOR_URL="$BUILDPACK_VENDOR_URL"
-elif [[ -n ${USE_STAGING_BINARIES} ]]; then
-    VENDOR_URL="$USE_STAGING_BINARIES/$STACK"
+if [[ "${STACK}" == "cedar-14" ]]; then
+  # Since Cedar-14 is EOL on 2020-11-02 it's not worth migrating it to the new bucket.
+  DEFAULT_S3_BASE_URL='https://lang-python.s3.amazonaws.com'
+else
+  DEFAULT_S3_BASE_URL='https://heroku-buildpack-python.s3.amazonaws.com'
 fi
-export VENDOR_URL
+S3_BASE_URL="${BUILDPACK_S3_BASE_URL:-${DEFAULT_S3_BASE_URL}}"
+# This has to be exported since it's used by the geo-libs step which is run in a subshell.
+# TODO: Stop exporting once the geo-libs step is removed or no longer uses `sub_env`.
+export S3_BASE_URL

 # Default Python Versions
 # shellcheck source=bin/default_pythons
 source "$BIN_DIR/default_pythons"

 # Supported Python Branches
+PY39="python-3.9"
 PY38="python-3.8"
 PY37="python-3.7"
 PY36="python-3.6"
@@ -63,37 +69,13 @@ PYPY36="pypy3.6"

 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
-# If pip doesn't match this version (the version we install), run the installer.
-PIP_UPDATE="20.0.2"
-
-for file in "$BUILD_DIR/runtime.txt" "$CACHE_DIR/.heroku/python-version" ; do
-    [ -f "$file" ] || continue
-
-    version=$(tr -d '[:space:]' < "$file")
-
-    case "$version" in "$PY34"*)
-        # Python 3.4 support was dropped in pip >= 19.2.
-        PIP_UPDATE="19.1.1"
-        break
-        ;;
-    esac
-done
-
-if [[ -f "$BUILD_DIR/Pipfile" ]]; then
-    # Do not force pipenv users to re-install pipenv locally.
-    PIP_UPDATE="9.0.2"
-fi
-
-export DEFAULT_PYTHON_STACK PIP_UPDATE
-export PY37 PY36 PY35 PY27 PY34

 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.
 # The `warnings` subscript then greps through this for common problems and guides
 # the user towards resolution of known issues.
 WARNINGS_LOG=$(mktemp)
-export WARNINGS_LOG
-export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
+RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION

 # The buildpack ships with a few executable tools (e.g. pip-grep, etc).
 # This installs them into the path, so we can execute them directly.
@@ -159,6 +141,10 @@ export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PAT
 export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
 export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH

+# Global pip options (https://pip.pypa.io/en/stable/user_guide/#environment-variables).
+# Disable pip's warnings about EOL Python since we show our own.
+export PIP_NO_PYTHON_VERSION_WARNING=1
+
 # The Application Code
 # --------------------

@@ -175,7 +161,7 @@ cd "$BUILD_DIR"
 # - Once the build is complete, `~/.heroku/{known-paths}` is copied back into the cache.

 # Create the cache directory, if it doesn't exist.
-mkdir -p "$CACHE_DIR"
+mkdir -p "$CACHE_DIR/.heroku"

 # Restore old artifacts from the cache.
 mkdir -p .heroku
@@ -207,7 +193,7 @@ source "$BIN_DIR/steps/hooks/pre_compile"
 # continue to use that version of Python in perpituity (warnings will be raised if
 # they are out–of–date).
 if [ -f "$CACHE_DIR/.heroku/python-version" ]; then
-  DEFAULT_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
+  CACHED_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
 fi

 # We didn't always record the stack version. This code is in place because of that.
@@ -217,8 +203,6 @@ else
  CACHED_PYTHON_STACK=$STACK
 fi

-export CACHED_PYTHON_STACK
-
 # Pipenv Python version support.
 # Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version).
 # Convert it to a runtime.txt file.
@@ -226,9 +210,14 @@ export CACHED_PYTHON_STACK
 # shellcheck source=bin/steps/pipenv-python-version
 source "$BIN_DIR/steps/pipenv-python-version"

-# If no runtime was provided by the user, assume the default Python runtime version.
-if [ ! -f runtime.txt ]; then
-  echo "$DEFAULT_PYTHON_VERSION" > runtime.txt
+if [[ -f runtime.txt ]]; then
+  mcount "version.reason.python.specified"
+elif [[ -n "${CACHED_PYTHON_VERSION:-}" ]]; then
+  mcount "version.reason.python.cached"
+  echo "${CACHED_PYTHON_VERSION}" > runtime.txt
+else
+  mcount "version.reason.python.default"
+  echo "${DEFAULT_PYTHON_VERSION}" > runtime.txt
 fi

 # Create the directory for .profile.d, if it doesn't exist.
@@ -259,17 +248,8 @@ mtime "python.install.time" "${start}"
 # shellcheck source=bin/steps/pipenv
 source "$BIN_DIR/steps/pipenv"

-# Uninstall removed dependencies with Pip.
-# The buildpack will automatically remove any declared dependencies (in requirements.txt)
-# that were explicitly removed. This machinery is a bit complex, but it is not complicated.
-(( start=$(nowms) ))
-# shellcheck source=bin/steps/pip-uninstall
-source "$BIN_DIR/steps/pip-uninstall"
-mtime "pip.uninstall.time" "${start}"
-
 # If no requirements.txt file given, assume `setup.py develop` is intended.
 # This allows for people to ship a setup.py application to Heroku
-# (which is rare, but I vouch that it should work!)

 if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
  echo "-e ." > requirements.txt
@@ -333,7 +313,7 @@ mtime "nltk.download.time" "${start}"
 # and copying it into the proper place (the logical place to do this was early, but it must be done here).
 # In CI, $BUILD_DIR is /app.
 if [[ ! "$BUILD_DIR" == "/app" ]]; then
-  rm -fr "$BUILD_DIR/.heroku/src"
+  rm -rf "$BUILD_DIR/.heroku/src"
  deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src"
 fi

@@ -1,21 +1,17 @@
 #!/usr/bin/env bash

-DEFAULT_PYTHON_VERSION="python-3.6.10"
-LATEST_38="python-3.8.1"
-LATEST_37="python-3.7.6"
-LATEST_36="python-3.6.10"
-LATEST_35="python-3.5.7"
-LATEST_34="python-3.4.10"
-LATEST_27="python-2.7.17"
-PYPY_36="pypy3.6-7.2.0"
-PYPY_27="pypy2.7-7.2.0"
+# Disable unused env var warning, since shellcheck doesn't take into account
+# that this file is sourced. We don't want to use export since it exposes
+# the env vars to subprocesses.
+# shellcheck disable=2034

-export DEFAULT_PYTHON_VERSION \
-    LATEST_38 \
-    LATEST_37 \
-    LATEST_36 \
-    LATEST_35 \
-    LATEST_34 \
-    LATEST_27 \
-    PYPY_36 \
-    PYPY_27
+DEFAULT_PYTHON_VERSION="python-3.6.12"
+LATEST_39="python-3.9.0"
+LATEST_38="python-3.8.6"
+LATEST_37="python-3.7.9"
+LATEST_36="python-3.6.12"
+LATEST_35="python-3.5.10"
+LATEST_34="python-3.4.10"
+LATEST_27="python-2.7.18"
+LATEST_PYPY_36="pypy3.6-7.3.2"
+LATEST_PYPY_27="pypy2.7-7.3.2"
@@ -0,0 +1,21 @@
+# Python Buildpack Install Steps
+
+TODO: Add context on Python install steps, such as why symlinking vs copying
+
+## Installing Python packages using Pip
+
+### Convention: Use `python` process to invoke Pip
+
+We don't use this convention (yet) but this is an upcoming change being considered.
+
+This is a bigger concern on Windows than it is in Linux environments, but an
+emerging convention in the Python community is to invoke pip using:
+
+```
+python3 -m pip [options]
+```
+
+Invoking pip this way ensures correct location - python knows where these
+packages are stored because it put them there (defaults to Python's pathing info).
+
+All normal command line options are available using this method.
@@ -10,7 +10,7 @@
 # This script is invoked by [`bin/compile`](/).

 # The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
+VENDORED_GDAL="${S3_BASE_URL}/${STACK}/libraries/vendor/gdal.tar.gz"

 PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"

@@ -21,7 +21,14 @@ source "$BIN_DIR/utils"
 # If GDAL exists within requirements, use vendored gdal.
 if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then

-  if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then
+  if [[ ! -f ".heroku/vendor/bin/gdalserver" && "${STACK}" == 'heroku-20' ]]; then
+    puts-warn "The buildpack's built-in GDAL functonality is not supported on Heroku-20."
+    puts-warn "Please use this buildpack instead: https://github.com/heroku/heroku-geo-buildpack"
+
+  elif [[ ! -f ".heroku/vendor/bin/gdalserver" ]]; then
+    puts-warn "The vendored GDAL package in the Heroku Python Buildpack now deprecated."
+    puts-warn "To enable GDAL use an alternative buildpack is available here - https://github.com/heroku/heroku-geo-buildpack"
+
    echo "-----> Noticed GDAL. Bootstrapping gdal."
    mkdir -p .heroku/vendor
    # Download and extract cryptography into target vendor directory.
@@ -10,9 +10,9 @@
 # This script is invoked by [`bin/compile`](/).

 # The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${VENDOR_URL}/libraries/vendor/gdal.tar.gz"
-VENDORED_GEOS="${VENDOR_URL}/libraries/vendor/geos.tar.gz"
-VENDORED_PROJ="${VENDOR_URL}/libraries/vendor/proj.tar.gz"
+VENDORED_GDAL="${S3_BASE_URL}/${STACK}/libraries/vendor/gdal.tar.gz"
+VENDORED_GEOS="${S3_BASE_URL}/${STACK}/libraries/vendor/geos.tar.gz"
+VENDORED_PROJ="${S3_BASE_URL}/${STACK}/libraries/vendor/proj.tar.gz"

 PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"

@@ -20,10 +20,16 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 # shellcheck source=bin/utils
 source "$BIN_DIR/utils"

-# If GDAL exists within requirements, use vendored gdal.
-if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
+if [[ "$BUILD_WITH_GEO_LIBRARIES" && "${STACK}" == 'heroku-20' ]]; then
+  puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality is not supported on Heroku-20."
+  puts-warn "Please use this buildpack for GDAL, GEOS and PROJ: https://github.com/heroku/heroku-geo-buildpack"
+  puts-warn "To hide this message, unset the BUILD_WITH_GEO_LIBRARIES variable using: heroku config:unset BUILD_WITH_GEO_LIBRARIES"
+elif [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
  mcount "buildvar.BUILD_WITH_GEO_LIBRARIES"

+  puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."
+  puts-warn "An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
+
  if [ ! -f ".heroku/vendor/bin/proj" ]; then
    echo "-----> Bootstrapping gdal, geos, proj."
    mkdir -p .heroku/vendor
@@ -38,9 +38,16 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
        mcount "failure.none-version"
    fi

+    if grep -qi '^django==1.*' requirements.txt; then
+        puts-warn "Your Django version is nearing the end of its community support."
+        puts-warn "Upgrade to continue to receive security updates and for the best experience with Django."
+        puts-warn "For more information, check out https://www.djangoproject.com/download/#supported-versions"
+    fi
+
    if [ ! -f "$BUILD_DIR/.heroku/python/bin/pip" ]; then
        exit 1
    fi
+
    /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
    PIP_STATUS="${PIPESTATUS[0]}"
    set -e
@@ -52,12 +59,9 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
        exit 1
    fi

-    # Smart Requirements handling
    cp requirements.txt .heroku/python/requirements-declared.txt
    /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt

-    echo
-
    # Install test dependencies, for CI.
    if [ "$INSTALL_TEST" ]; then
        if [[ -f "$1/requirements-test.txt" ]]; then
@@ -16,7 +16,7 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
      mcount "failure.bad-requirements"
    fi

-    rm -fr requirements-declared.txt
+    rm -rf requirements-declared.txt

    if [[ -s .heroku/python/requirements-stale.txt ]]; then
      puts-step "Uninstalling stale dependencies"
@@ -9,10 +9,7 @@ set -e
 if [[ -f Pipfile.lock ]]; then
    if [[ -f .heroku/python/Pipfile.lock.sha256 ]]; then
        if [[ $(openssl dgst -sha256 Pipfile.lock) == $(cat .heroku/python/Pipfile.lock.sha256) ]]; then
-            # Measure that we're using Pipenv.
-            mcount "tool.pipenv"
-
-            # Don't skip installation of there are git deps.
+            # Don't skip installation if there are git deps.
            if ! grep -q 'git' Pipfile.lock; then
                echo "Skipping installation, as Pipfile.lock hasn't changed since last deploy." | indent

@@ -27,6 +27,12 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
                if [ "$PYTHON" = 3.7 ]; then
                    echo "$LATEST_37" > "$BUILD_DIR/runtime.txt"
                fi
+                if [ "$PYTHON" = 3.8 ]; then
+                    echo "$LATEST_38" > "$BUILD_DIR/runtime.txt"
+                fi
+                if [ "$PYTHON" = 3.9 ]; then
+                    echo "$LATEST_39" > "$BUILD_DIR/runtime.txt"
+                fi
            fi


@@ -15,7 +15,7 @@ if [[ "$STACK" != "cedar-14" ]]; then
 fi

 # The location of the pre-compiled libmemcached binary.
-VENDORED_MEMCACHED="${VENDOR_URL}/libraries/vendor/libmemcache.tar.gz"
+VENDORED_MEMCACHED="${S3_BASE_URL}/${STACK}/libraries/vendor/libmemcache.tar.gz"

 # Syntax sugar.
 # shellcheck source=bin/utils
@@ -5,9 +5,10 @@ runtime-fixer runtime.txt
 PYTHON_VERSION=$(cat runtime.txt)

 # The location of the pre-compiled python binary.
-VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"
+VENDORED_PYTHON="${S3_BASE_URL}/${STACK}/runtimes/${PYTHON_VERSION}.tar.gz"

 SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
+SECURITY_UPDATE_PYPY="The PyPy project has released a security update! Please consider upgrading to"

 ONLY_SUPPORTED_2_VERSION="Only the latest version of Python 2 is supported on the platform. Please consider upgrading to"

@@ -15,6 +16,12 @@ PYTHON_2_EOL_UPDATE="Python 2 has reached it's community EOL. Upgrade your Pytho

 # check if runtime exists
 if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
+  if [[ "$PYTHON_VERSION" == $PY39* ]]; then
+    if [ "$PYTHON_VERSION" != "$LATEST_39" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_39"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
  if [[ "$PYTHON_VERSION" == $PY38* ]]; then
    # do things to alert the user of security release available
    if [ "$PYTHON_VERSION" != "$LATEST_38" ]; then
@@ -51,11 +58,9 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
    fi
  fi
  if [[ "$PYTHON_VERSION" == $PY27* ]]; then
+    puts-warn "$PYTHON_2_EOL_UPDATE"
+    echo "       Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq"
    # security update note
-    if [[ "$(date "+%Y")" -gt "2019" ]]; then
-      puts-warn "$PYTHON_2_EOL_UPDATE"
-      echo "       Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq"
-    fi
    if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then
      puts-warn "$ONLY_SUPPORTED_2_VERSION" "$LATEST_27"
      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
@@ -63,15 +68,15 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
  fi
  if [[ "$PYTHON_VERSION" == $PYPY27* ]]; then
    # security update note
-    if [ "$PYTHON_VERSION" != "$PYPY_27" ]; then
-      puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_27}?"
+    if [ "$PYTHON_VERSION" != "$LATEST_PYPY_27" ]; then
+      puts-warn "$SECURITY_UPDATE_PYPY" "$LATEST_PYPY_27"
      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
    fi
  fi
  if [[ "$PYTHON_VERSION" == $PYPY36* ]]; then
    # security update note
-    if [ "$PYTHON_VERSION" != "$PYPY_36" ]; then
-      puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_36}?"
+    if [ "$PYTHON_VERSION" != "$LATEST_PYPY_36" ]; then
+      puts-warn "$SECURITY_UPDATE_PYPY" "$LATEST_PYPY_36"
      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
    fi
  fi
@@ -81,28 +86,40 @@ else
  exit 1
 fi

+mcount "version.python.${PYTHON_VERSION}"
+
 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
    puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache"
-    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
-fi
-
-# need to clear the cache for first time installing SQLite3,
-# since the version is changing and could lead to runtime errors
-# with compiled extensions.
-if [ -d .heroku/python ] && [ ! -f .heroku/python-sqlite3-version ] && python_sqlite3_check "$PYTHON_VERSION"; then
-  puts-step "Need to update SQLite3, clearing cache"
-  rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
+    rm -rf .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
 fi

 if [ -f .heroku/python-version ]; then
  if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then
      puts-step "Found $(cat .heroku/python-version), removing"
-      rm -fr .heroku/python
+      rm -rf .heroku/python
  else
    SKIP_INSTALL=1
  fi
 fi

+# Check if we should reinstall python dependencies
+if [[ ! -f "$CACHE_DIR/.heroku/requirements.txt" ]]; then
+  # IF there's no cached dependencies, update cached version of requirements.txt
+  # This should only run for new apps and first deploys after this update
+  cp -R "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt"
+else
+  # IF there IS a cached directory, check for differences with the new one
+  if ! diff "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt" &> /dev/null; then
+    puts-step "Requirements file has been changed, clearing cached dependencies"
+    # if there are any differences, clear the Python cache
+    # Installing Python over again does not take noticably more time
+    cp -R "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt"
+    rm -rf .heroku/python
+    unset SKIP_INSTALL
+  else
+    puts-step "No change in requirements detected, installing from cache"
+  fi
+fi

 if [ ! "$SKIP_INSTALL" ]; then
    puts-step "Installing $PYTHON_VERSION"
@@ -110,8 +127,6 @@ if [ ! "$SKIP_INSTALL" ]; then
    # Prepare destination directory.
    mkdir -p .heroku/python

-    mcount "version.python.$PYTHON_VERSION"
-
    if ! curl "${VENDORED_PYTHON}" -s | tar zxv -C .heroku/python &> /dev/null; then
      puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)."
      puts-warn "Aborting.  More info: https://devcenter.heroku.com/articles/python-support"
@@ -121,34 +136,58 @@ if [ ! "$SKIP_INSTALL" ]; then
  # Record for future reference.
  echo "$PYTHON_VERSION" > .heroku/python-version
  echo "$STACK" > .heroku/python-stack
-  FRESH_PYTHON=true

  hash -r
 fi

-# Heroku uses the get-pip utility maintained by the Python community to vendor Pip.
-# https://github.com/pypa/get-pip
-GETPIP_URL="https://lang-python.s3.amazonaws.com/etc/get-pip.py"
-GETPIP_PY="${TMPDIR:-/tmp}/get-pip.py"
-
-if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then
-    mcount "failure.python.get-pip"
-    echo "Failed to pull down get-pip"
-    exit 1
-fi
-
-# If Pip isn't up to date:
-if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
-
-  puts-step "Installing pip"
-
-  # Remove old installations.
-  rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
-  rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
-
-  /app/.heroku/python/bin/python "$GETPIP_PY" pip=="$PIP_UPDATE" &> /dev/null
-  /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
-fi
-
 set -e
+
+PIP_VERSION='20.1.1'
+# Must use setuptools <47.2.0 until we fix:
+# https://github.com/heroku/heroku-buildpack-python/issues/1006
+SETUPTOOLS_VERSION='47.1.1'
+WHEEL_VERSION='0.34.2'
+
+if [[ "${PYTHON_VERSION}" == ${PY34}* ]]; then
+  # Python 3.4 support was dropped in pip 19.2+, setuptools 44+ and wheel 0.34+.
+  PIP_VERSION='19.1.1'
+  SETUPTOOLS_VERSION='43.0.0'
+  WHEEL_VERSION='0.33.6'
+elif [[ "${PYTHON_VERSION}" == ${PY27}* || "${PYTHON_VERSION}" == ${PYPY27}* ]]; then
+  # Python 2.7 support was dropped in setuptools 45+.
+  SETUPTOOLS_VERSION='44.1.1'
+fi
+
+# We don't use get-pip.py, since:
+#  - it uses `--force-reinstall`, which is unnecessary here and slows down repeat builds
+#  - it means downloading pip twice (once embedded in get-pip.py, and again during
+#    the install, since get-pip.py can't install the embedded version directly)
+#  - we would still have to manage several versions of get-pip.py, to support older Pythons.
+# Instead, we use the pip wheel to install itself, using the method described here:
+# https://github.com/pypa/pip/issues/2351#issuecomment-69994524
+PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl"
+PIP_WHEEL_URL="${S3_BASE_URL}/common/${PIP_WHEEL_FILENAME}"
+PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}"
+
+if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then
+  mcount "failure.python.download-pip"
+  puts-warn "Failed to download pip"
+  exit 1
+fi
+
+if [[ -f "$BUILD_DIR/Pipfile" ]]; then
+  # The buildpack is pinned to old pipenv, which requires older pip.
+  # Pip 9.0.2 doesn't support installing itself from a wheel, so we have to use split
+  # versions here (ie: installer pip version different from target pip version).
+  PIP_VERSION='9.0.2'
+  PIP_TO_INSTALL="pip==${PIP_VERSION}"
+else
+  PIP_TO_INSTALL="${PIP_WHEEL}"
+fi
+
+puts-step "Installing pip ${PIP_VERSION}, setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}"
+
+/app/.heroku/python/bin/python "${PIP_WHEEL}/pip" install --quiet --disable-pip-version-check --no-cache \
+  "${PIP_TO_INSTALL}" "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}"
+
 hash -r
@@ -67,7 +67,6 @@ buildpack_sqlite3_install() {
  puts-step "Installing SQLite3"

  if sqlite3_install "$BUILD_DIR/.heroku/python" ; then
-    echo "Sqlite3 successfully installed."
    mcount "success.python.sqlite3"
  else
    echo "Sqlite3 failed to install."
@@ -1,57 +1,80 @@
 # Python Buildpack Binaries

-## Building the Docker Images
-
-**After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack:
-
-    $ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
-    $ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-    $ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-
-## Using the Image
-
-You can e.g. `bash` into each of the images you built using their tag:
-
-    docker run --rm -ti heroku-python-build-cedar-14 bash
-    docker run --rm -ti heroku-python-build-heroku-16 bash
-    docker run --rm -ti heroku-python-build-heroku-18 bash
-
-You then have a shell where you can run `bob build`, `bob deploy`, and so forth. You can of course also invoke these programs directly with `docker run`:
-
-    docker run --rm -ti heroku-python-build-heroku-18 bob build runtimes/python-2.7.15
-
-In order to `bob deploy`, AWS credentials must be set up, as well as name and prefix of your custom S3 bucket (unless you're deploying to the Heroku production buckets that are pre-defined in each `Dockerfile`); see next section for details.
+The binaries for this buildpack are built in Docker containers based on the Heroku stack image.

 ## Configuration

-File `dockerenv.default` contains a list of required env vars; most of these have default values defined in `Dockerfile`. You can copy this file to a location outside the buildpack and modify it with the values you desire and pass its location with `--env-file`, or pass the env vars to `docker run` using `--env`.
+In order to publish binaries AWS credentials must be passed to the build container.
+If you are testing only the build (ie: `bob build`), these are optional.

-Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCKET`, `S3_PREFIX`, and `S3_REGION`. If you're building your own packages, you'll likely want to change `S3_BUCKET` and `S3_PREFIX` to match your info. Instead of setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` into that file, you may also pass them to `docker run` through the environment, or explicitly using `--env`, in order to prevent accidental commits of credentials.
+In addition, unless you are building the official binaries for Heroku (which use the defaults
+specified in each `Dockerfile`), you will need to override `S3_BUCKET` and `S3_PREFIX` to
+match your own S3 bucket/use case.

-### Passing AWS credentials to the container
+If you only need to set AWS credentials, you can do so by setting the environment variables
+`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` before calling the make commands.

-If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file.
+For example:

-#### Passing credentials explicitly
+```bash
+set +o history # Disable bash history
+export AWS_ACCESS_KEY_ID=...
+export AWS_SECRET_ACCESS_KEY=...
+set -o history # Re-enable bash history
+make ...
+```

-    docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash
+If you need to override the default S3 bucket, or would prefer not to use credentials via
+environment variables, then you need to instead use a Docker env file like so:

-#### Passing credentials through  the environment
+1. Copy the `builds/dockerenv.default` env file to a location outside the buildpack repository.
+2. Edit the new file, adding at a minimum the values for the variables
+   `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (see Docker
+   [env-file documentation](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file)).
+3. Pass the path of the file to the make commands using `ENV_FILE`. For example:

-The two environment variables `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY` are defined in `builds/dockerenv.default`, without values. This will cause Docker to "forward" values for these variables from the current environment, so you can pass them in:
+   ```bash
+   make ... ENV_FILE=~/.dockerenv.python-buildpack
+   ```

-    AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+## Launching an interactive build environment

-or
+To start an interactive version of the build environment (ideal for development) use the
+`buildenv` make target, passing in the desired `STACK` name. For example:

-    export AWS_ACCESS_KEY_ID=...
-    export AWS_SECRET_ACCESS_KEY=...
-    docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+```bash
+make buildenv STACK=heroku-18
+```

-#### Passing credentials through a separate env file
+This will create the builder docker image based on the latest image for that stack, and
+then start a bash shell where you can run `bob build`, `bob deploy`, and so forth.

-This method is the easiest for users who want to build packages in their own S3 bucket, as they will have to adjust the `S3_BUCKET` and `S3_PREFIX` environment variable values anyway from their default values.
+The `builds/` directory is bind-mounted into the running container, so local build formula
+changes will appear there immediately without the need to rebuild the image.

-For this method, it is important to keep the credentials file in a location outside the buildpack, so that your credentials aren't accidentally committed. Copy `builds/dockerenv.default` **to a safe location outside the buildpack directory**, and insert your values for `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY`.
+## Bulk deploying runtimes

-    docker run --rm -ti --env-file=../SOMEPATHOUTSIDE/s3.env heroku-python-build-heroku-18 bash
+When a new Python version is released, binaries have to be generated for multiple stacks.
+To automate this, use the `deploy-runtimes` make target, which will ensure the builder
+image is up to date, and then run `bob deploy` for each runtime-stack combination.
+
+The build formula name(s) are passed using `RUNTIMES`, like so:
+
+```bash
+make deploy-runtimes RUNTIMES='python-X.Y.Z'
+```
+
+By default this will deploy to all supported stacks (see `STACKS` in `Makefile`),
+but this can be overridden using `STACKS`:
+
+```bash
+make deploy-runtimes RUNTIMES='python-X.Y.Z' STACKS='heroku-16 heroku-18'
+```
+
+Multiple runtimes can also be specified (useful for when adding a new stack), like so:
+
+```bash
+make deploy-runtimes RUNTIMES='python-A.B.C python-X.Y.Z' STACKS='heroku-20'
+```
+
+Note: Both `RUNTIMES` and `STACKS` are space delimited.
@@ -1,15 +1,21 @@
 FROM heroku/cedar:14

-WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
    S3_BUCKET="lang-python" \
    S3_PREFIX="cedar-14/" \
-    DEBIAN_FRONTEND=noninteractive \
    STACK="cedar-14"

-RUN apt-get update && apt-get install -y python-pip libsqlite3-dev realpath && rm -rf /var/lib/apt/lists/*
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        realpath \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app

 COPY requirements.txt /app/
-RUN pip install -r /app/requirements.txt
+# Can't use `--disable-pip-version-check --no-cache-dir` since not supported by Ubuntu 14.04's pip.
+RUN pip3 install -r /app/requirements.txt

 COPY . /app
@@ -1,5 +1,8 @@
+# Since no values are specified here, these variables will be read from the environment at run time:
+# https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file
 AWS_ACCESS_KEY_ID
 AWS_SECRET_ACCESS_KEY
-S3_BUCKET
-S3_PREFIX
-S3_REGION
+
+# Uncomment these if you need to override the default S3 bucket and/or path prefixes.
+# S3_BUCKET
+# S3_PREFIX
@@ -1,15 +1,20 @@
 FROM heroku/heroku:16-build

-WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
-    S3_BUCKET="lang-python" \
+    S3_BUCKET="heroku-buildpack-python" \
    S3_PREFIX="heroku-16/" \
-    DEBIAN_FRONTEND=noninteractive \
    STACK="heroku-16"

-RUN apt-get update && apt-get install -y python-pip libsqlite3-dev && rm -rf /var/lib/apt/lists/*
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        python3-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app

 COPY requirements.txt /app/
-RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt

 COPY . /app
@@ -1,15 +1,20 @@
-FROM heroku/heroku:18-build.v16
+FROM heroku/heroku:18-build

-WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
-    S3_BUCKET="lang-python" \
+    S3_BUCKET="heroku-buildpack-python" \
    S3_PREFIX="heroku-18/" \
-    DEBIAN_FRONTEND=noninteractive \
    STACK="heroku-18"

-RUN apt-get update && apt-get install --no-install-recommends -y python-pip-whl=9.0.1-2 python-pip=9.0.1-2 python-setuptools python-wheel libsqlite3-dev && rm -rf /var/lib/apt/lists/*
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        python3-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app

 COPY requirements.txt /app/
-RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt

 COPY . /app
@@ -0,0 +1,20 @@
+FROM heroku/heroku:20-build
+
+ENV WORKSPACE_DIR="/app/builds" \
+    S3_BUCKET="heroku-buildpack-python" \
+    S3_PREFIX="heroku-20/" \
+    STACK="heroku-20"
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        libsqlite3-dev \
+        python3-pip \
+        python3-setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt /app/
+RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+
+COPY . /app
@@ -19,4 +19,4 @@ make install

 # Cleanup
 cd ..
-rm -fr sqlite
+rm -rf sqlite
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Skip the build for heroku-16.
-if [[ $S3_PREFIX == "heroku-16" ]]; then
-    exit 0
-fi
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building libffi…"
-
-SOURCE_TARBALL='ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz'
-
-curl -L $SOURCE_TARBALL | tar x
-
-cd libffi-3.1
-./configure --prefix=$OUT_PREFIX --disable-static &&
-make
-make install
-
-# Cleanup
-cd ..
@@ -3,9 +3,9 @@

 OUT_PREFIX=$1

-# Skip the build for heroku-16.
-if [[ $S3_PREFIX == "heroku-16" ]]; then
-    exit 0
+if [[ $S3_PREFIX != "cedar-14" ]]; then
+    echo "libmemcached only needs to be built for cedar-14, since newer stacks include it in the base image"
+    exit 1
 fi

 # fail hard
@@ -13,11 +13,11 @@ dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full ver
 dep_package=${dep_name}-v${dep_version} # it's always "pypy2-…"
 dep_dirname=${dep_package}-linux64
 dep_archive_name=${dep_dirname}.tar.bz2
-dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+dep_url="https://downloads.python.org/pypy/${dep_archive_name}"

 echo "Building PyPy…"
 echo "${dep_url}"

-curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+curl -fL "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name

 ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python"
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy2.7
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy2.7
@@ -13,11 +13,11 @@ dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full ver
 dep_package=${dep_name}${dep_version_prefix:-}-v${dep_version}${dep_version_suffix:-}
 dep_dirname=${dep_package}-linux64
 dep_archive_name=${dep_dirname}.tar.bz2
-dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+dep_url="https://downloads.python.org/pypy/${dep_archive_name}"

 echo "Building PyPy3…"
 echo "${dep_url}"

-curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+curl -fL "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name

 ln "$OUT_PREFIX/bin/pypy3" "$OUT_PREFIX/bin/python"
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy3.6
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy3.6
@@ -1,27 +1,4 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/

-OUT_PREFIX=$1
-BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
-export BIN_DIR
-
-# shellcheck source=bin/utils
-source "$BIN_DIR/steps/sqlite3"
-
-sqlite3_version
-echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
-sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
-
-echo "Building Python…"
-SOURCE_TARBALL='https://python.org/ftp/python/2.7.17/Python-2.7.17.tgz'
-curl -L $SOURCE_TARBALL | tar xz
-mv Python-2.7.17 src
-cd src
-
-./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
-make
-make install
-
-# Remove unneeded test directories, similar to the official Docker Python images:
-# https://github.com/docker-library/python
-find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+source $(dirname $0)/python2
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python2
@@ -1,32 +1,4 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/

-OUT_PREFIX=$1
-BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
-export BIN_DIR
-
-# shellcheck source=bin/utils
-source "$BIN_DIR/steps/sqlite3"
-
-sqlite3_version
-echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
-sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
-
-echo "Building Python…"
-SOURCE_TARBALL='https://python.org/ftp/python/3.7.2/Python-3.7.2.tgz'
-curl -L $SOURCE_TARBALL | tar xz
-mv Python-3.7.2 src
-cd src
-
-./configure --prefix=$OUT_PREFIX --with-ensurepip=no
-make
-make install
-
-# Remove unneeded test directories, similar to the official Docker Python images:
-# https://github.com/docker-library/python
-find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
-
-# Remove spare /
-LOCATION=${OUT_PREFIX%?}
-
-ln $LOCATION/bin/python3 $LOCATION/bin/python
+source $(dirname $0)/python3
@@ -1,32 +1,4 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/

-OUT_PREFIX=$1
-BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
-export BIN_DIR
-
-# shellcheck source=bin/utils
-source "$BIN_DIR/steps/sqlite3"
-
-sqlite3_version
-echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
-sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
-
-echo "Building Python…"
-SOURCE_TARBALL='https://python.org/ftp/python/3.4.9/Python-3.4.9.tgz'
-curl -L $SOURCE_TARBALL | tar xz
-mv Python-3.4.9 src
-cd src
-
-./configure --prefix=$OUT_PREFIX --with-ensurepip=no
-make
-make install
-
-# Remove unneeded test directories, similar to the official Docker Python images:
-# https://github.com/docker-library/python
-find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
-
-# Remove spare /
-LOCATION=${OUT_PREFIX%?}
-
-ln $LOCATION/bin/python3 $LOCATION/bin/python
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3
@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-sudo apt-get -qq update
-sudo apt-get install software-properties-common
-curl --fail --retry 3 --retry-delay 1 --connect-timeout 3 --max-time 30 https://cli-assets.heroku.com/install-ubuntu.sh | sh
@@ -14,9 +14,9 @@ case "$choice" in
  * ) exit 1;;
 esac

-originMaster=$(git rev-parse origin/master)
-echo "Tagging commit $originMaster with $newVersion... "
-git tag "$newVersion" "${originMaster:?}"
+originMain=$(git rev-parse origin/main)
+echo "Tagging commit $originMain with $newVersion... "
+git tag "$newVersion" "${originMain:?}"
 git push origin refs/tags/$newVersion

 heroku buildpacks:publish "$BP_NAME" "$newVersion"
@@ -31,7 +31,7 @@ if [ $(git tag | grep -q latest-version) ]; then
    echo "Updating latest-version tag"
    git tag -d latest-version
    git push origin :latest-version
-    git tag latest-version "${originMaster:?}"
+    git tag latest-version "${originMain:?}"
    git push --tags
 fi

@@ -1,5 +1,6 @@
 {
  "python": [
-    "heroku/python-getting-started"
+    "heroku/python-getting-started",
+    "sharpstone/python_default"
  ]
 }
@@ -1,3 +1,5 @@
 ---
 - - "./repos/python/python-getting-started"
-  - f56b90499ec11e1c9576da2f8c7331300e189db3
+  - main
+- - "./repos/python/python_default"
+  - ca947f69027b2a30be5d26f9a42f25e54f4d7a1a
@@ -1,3 +1,6 @@
+# Dependencies for generating/publishing Python binaries.
+bob-builder==0.0.19
+
+# Sub-dependencies of bob-builder.
+boto==2.49.0
 docopt==0.6.2
-bob-builder
-boto==2.48.0
@@ -0,0 +1,33 @@
+require_relative '../spec_helper'
+
+describe "Heroku CI" do
+  it "works" do
+    before_deploy = Proc.new do
+      File.open("app.json", "w+") do |f|
+        f.puts <<~EOM
+          {
+            "environments": {
+              "test": {
+                "scripts": {
+                  "test": "nosetests"
+                }
+              }
+            }
+          }
+        EOM
+      end
+
+      run!("echo nose >> requirements.txt")
+    end
+
+    Hatchet::Runner.new("python_default", before_deploy: before_deploy).run_ci do |test_run|
+      expect(test_run.output).to match("Downloading nose")
+      expect(test_run.output).to match("OK")
+
+      test_run.run_again
+
+      expect(test_run.output).to match("installing from cache")
+      expect(test_run.output).to_not match("Downloading nose")
+    end
+  end
+end
@@ -1,10 +1,72 @@
 require_relative '../spec_helper'

-describe "Python!!!!!!!!!!!" do
-  it "🐍" do
-    Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app|
-      expect(app.output).to           match(/Installing pip/)
-      expect(app.run('python -V')).to match(/3.6.10/)
+describe "Python" do
+  describe "cache" do
+    it "functions correctly" do
+      Hatchet::Runner.new("python_default").deploy do |app|
+        expect(app.output).to match(/Installing pip/)
+
+        expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies")
+        expect(app.output).to_not match("No change in requirements detected, installing from cache")
+        expect(app.output).to_not match("No such file or directory")
+        expect(app.output).to_not match("cp: cannot create regular file")
+
+        # Redeploy with changed requirements file
+        run!(%Q{echo "" >> requirements.txt})
+        run!(%Q{echo "pygments" >> requirements.txt})
+        run!(%Q{git add . ; git commit --allow-empty -m next})
+        app.push!
+
+        # Check the cache to have cleared
+        expect(app.output).to match("Requirements file has been changed, clearing cached dependencies")
+        expect(app.output).to_not match("No dependencies found, preparing to install")
+        expect(app.output).to_not match("No change in requirements detected, installing from cache")
+
+        # With no changes on redeploy, the cache should be present
+        run!(%Q{git commit --allow-empty -m next})
+        app.push!
+
+        expect(app.output).to match("No change in requirements detected, installing from cache")
+        expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies")
+        expect(app.output).to_not match("No dependencies found, preparing to install")
+      end
+    end
+  end
+
+  describe "python versions" do
+    let(:stack) { ENV["HEROKU_TEST_STACK"] || DEFAULT_STACK }
+    it "works with 3.7.6" do
+      version = "3.7.6"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack).deploy do |app|
+        expect(app.run('python -V')).to match(version)
+      end
+    end
+
+    it "works with 3.8.2" do
+      version = "3.8.2"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack).deploy do |app|
+        expect(app.run('python -V')).to match(version)
+      end
+    end
+
+    it "fails with a bad version" do
+      version = "3.8.2.lol"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack, allow_failure: true).deploy do |app|
+        expect(app.output).to match("not available for this stack")
+      end
+    end
+  end
+
+  it "getting started app has no relative paths" do
+    buildpacks = [
+      :default,
+      "https://github.com/sharpstone/force_absolute_paths_buildpack"
+    ]
+    Hatchet::Runner.new("python-getting-started", buildpacks: buildpacks).deploy do |app|
+      # Deploy works
    end
  end
 end
@@ -1,4 +1,4 @@
-ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/' + ENV['TRAVIS_REPO_SLUG'] + '.git'
+ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/heroku/heroku-buildpack-python.git'

 require 'rspec/core'
 require 'rspec/retry'
@@ -15,9 +15,10 @@ RSpec.configure do |config|
  end
 end

-if ENV['TRAVIS']
-  # Don't execute tests against "merge" commits
-  exit 0 if ENV['TRAVIS_PULL_REQUEST'] != 'false' && ENV['TRAVIS_BRANCH'] == 'master'
-end
+DEFAULT_STACK = 'heroku-18'

-DEFAULT_STACK = 'heroku-16'
+def run!(cmd)
+  out = `#{cmd}`
+  raise "Error running command #{cmd} with output: #{out}" unless $?.success?
+  return out
+end
@@ -0,0 +1,4 @@
+set -euo pipefail
+
+echo "post_compile ran!"
+echo "post_compile env: $(printenv | cut -d '=' -f 1 | sort | xargs)."
@@ -0,0 +1,4 @@
+set -euo pipefail
+
+echo "pre_compile ran!"
+echo "pre_compile env: $(printenv | cut -d '=' -f 1 | sort | xargs)."
@@ -1 +0,0 @@
-python-3.6.6
@@ -0,0 +1,2 @@
+pwned-passwords-django==1.4
+Django==2.1
@@ -0,0 +1 @@
+Django==1.11
@@ -6,4 +6,4 @@ verify_ssl = true
 requests = "*"

 [requires]
-python_full_version = "3.6.3"
+python_full_version = "3.7.8"
@@ -1,24 +1,11 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "22a052f4d1cfe6518b2f236fe45c3208c587a9ab1323bdd390632e27278b541e"
-        },
-        "host-environment-markers": {
-            "implementation_name": "cpython",
-            "implementation_version": "3.6.3",
-            "os_name": "posix",
-            "platform_machine": "x86_64",
-            "platform_python_implementation": "CPython",
-            "platform_release": "16.7.0",
-            "platform_system": "Darwin",
-            "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64",
-            "python_full_version": "3.6.3",
-            "python_version": "3.6",
-            "sys_platform": "darwin"
+            "sha256": "8a36860f0f9cb55716222098062cea5c5e0f8127cafb9d0c694de327bac9fbc0"
        },
        "pipfile-spec": 6,
        "requires": {
-            "python_full_version": "3.6.3"
+            "python_full_version": "3.7.8"
        },
        "sources": [
            {
@@ -1 +1 @@
-pypy2.7-7.2.0
+pypy2.7-7.3.2
@@ -0,0 +1 @@
+pypy2.7-7.3.1
@@ -1 +1 @@
-pypy3.6-7.2.0
+pypy3.6-7.3.2
@@ -0,0 +1 @@
+pypy3.6-7.3.1
@@ -1 +1 @@
-python-2.7.17
+python-2.7.18
@@ -1 +0,0 @@
-requests
@@ -1 +0,0 @@
-python-2.7.99
@@ -1 +0,0 @@
-requests
@@ -1 +0,0 @@
-flask
@@ -1 +0,0 @@
-python-3.4.99
@@ -1 +0,0 @@
-flask
@@ -1 +1 @@
-python-3.5.7
+python-3.5.10
@@ -1 +0,0 @@
-flask
@@ -1 +0,0 @@
-python-3.5.99
@@ -1 +0,0 @@
-flask
@@ -1 +1 @@
-python-3.6.8
+python-3.6.12
@@ -1 +0,0 @@
-flask
@@ -1 +0,0 @@
-python-3.6.99
@@ -1 +0,0 @@
-requests
@@ -1 +1 @@
-python-3.6.7
+python-3.6.11
@@ -1 +1 @@
-python-3.7.2
+python-3.7.9
@@ -1 +0,0 @@
-flask
--- a/Show More
+++ b/Show More