Release v178 (#1063)

Since it was released over the weekend:
https://www.python.org/downloads/release/python-3510/
https://docs.python.org/3.5/whatsnew/changelog.html#python-3-5-10

Closes @W-7835961@.

.git2gus/config.json

@@ -0,0 +1,6 @@
+{
+  "productTag": "a1aB0000000MR0RIAW",
+  "issueTypeLabels": { "gus: story": "USER STORY", "gus: bug": "BUG P3" },
+  "defaultBuild": "Heroku Unscheduled",
+  "statusWhenClosed": "CLOSED"
+}

.github/dependabot.yml

@@ -0,0 +1,15 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "c: dependencies"
+
+  - package-ecosystem: "bundler"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "c: dependencies"

.github/workflows/check_changelog.yml

@@ -1,14 +1,17 @@
 name: Check Changelog
 
 on:
- pull_request:
+  pull_request:
-  types: [opened, reopened, edited, synchronize]
+    types: [opened, reopened, edited, synchronize]
 
 jobs:
- build:
+  check:
-   runs-on: ubuntu-latest
+    runs-on: ubuntu-latest
-   steps:
+    if: |
-   - uses: actions/checkout@v1
+      !contains(github.event.pull_request.body, '[skip changelog]') &&
-   - name: Check that CHANGELOG is touched
+      !contains(github.event.pull_request.body, '[changelog skip]') &&
-     run: |
+      !contains(github.event.pull_request.body, '[skip ci]')
-       cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
+    steps:
+      - uses: actions/checkout@v1
+      - name: Check that CHANGELOG is touched
+        run: git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md

.pre-commit-config.yaml

@@ -1,7 +0,0 @@
-repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
-    hooks:
-    -   id: check-yaml
-    -   id: end-of-file-fixer
-    -   id: trailing-whitespace

.travis.yml

@@ -1,44 +1,41 @@
-language: ruby
+language: minimal
-dist: bionic
+dist: focal
-sudo: required
 branches:
   only:
-  - master
+  - main
-
-rvm:
-- 2.6.6
-before_script:
-  - gem install bundler -v 1.16.2
 
 script:
-  - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
+  - make test STACK="${STACK}" TEST_CMD="${TEST_CMD}"
-  - docker run --rm -e "STACK=cedar-14" travis-build-cedar-14 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-  - docker run --rm -e "STACK=heroku-16" travis-build-heroku-16 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-  - docker run --rm -e "STACK=heroku-18" travis-build-heroku-18 bash $TESTFOLDER
 
 jobs:
   include:
-  - stage: Bash linting (shellcheck)
+  - name: Bash linting (shellcheck)
-    sudo: false
     script: make check
-  - stage: Hatchet Integration
+  - name: Hatchet integration tests
-    if: env(TRAVIS_PULL_REQUEST_SLUG) = env(TRAVIS_REPO_SLUG)
+    if: env(HEROKU_API_USER) IS present AND env(HEROKU_API_KEY) IS present
-    name: Run Hatchet
+    language: ruby
-    script:
+    rvm:
+      - 2.7
+    before_script:
       - bundle exec hatchet ci:setup
+    script:
       - PARALLEL_SPLIT_TEST_PROCESSES=11 bundle exec parallel_split_test spec/hatchet/
 
 env:
-  matrix:
+  jobs:
-    - TESTFOLDER=test/run-deps
+    - STACK=cedar-14 TEST_CMD=test/run-deps
-    - TESTFOLDER=test/run-versions
+    - STACK=cedar-14 TEST_CMD=test/run-versions
-    - TESTFOLDER=test/run-features
+    - STACK=cedar-14 TEST_CMD=test/run-features
+
+    - STACK=heroku-16 TEST_CMD=test/run-deps
+    - STACK=heroku-16 TEST_CMD=test/run-versions
+    - STACK=heroku-16 TEST_CMD=test/run-features
+
+    - STACK=heroku-18 TEST_CMD=test/run-deps
+    - STACK=heroku-18 TEST_CMD=test/run-versions
+    - STACK=heroku-18 TEST_CMD=test/run-features
   global:
     - HATCHET_RETRIES=3
     - IS_RUNNING_ON_CI=true
     - HATCHET_APP_LIMIT=80
     - HATCHET_DEPLOY_STRATEGY=git
-    - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
-    - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=

CHANGELOG.md

@@ -1,72 +1,120 @@
 # Python Buildpack Changelog
 
-# 170 (2020-05-19)
+## Unreleased
+
+
+## v178 (2020-09-07)
+
+- Python 3.5.10 is now available (CPython) (#1062).
+
+## v177 (2020-08-18)
+
+- Python 3.6.12 and 3.7.9 are now available (CPython) (#1054).
+- The default Python version for new apps is now 3.6.12 (previously 3.6.11) (#1054).
+
+## v176 (2020-08-12)
+
+- Rebuild the Python 3.4.10 archives with the correct version of Python (#1048).
+- Fix the security update version check message for apps using PyPy (#1040).
+- Remove `vendor/test-utils` (#1043).
+
+## v175 (2020-08-05)
+
+- Update pip from 20.0.2 to 20.1.1 for Python 2.7 and Python 3.5+ (#1030).
+- Update setuptools from 39.0.1 to: (#1024)
+  - 44.1.1 for Python 2.7
+  - 43.0.0 for Python 3.4
+  - 47.1.1 for Python 3.5+
+- Switch the `heroku-buildpack-python` repository default branch from `master` to `main` (#1029).
+
+## v174 (2020-07-30)
+
+- For repeat builds, also manage the installed versions of setuptools/wheel, rather than just that of pip (#1007).
+- Install an explicit version of wheel rather than the latest release at the time (#1007).
+- Output the installed version of pip, setuptools and wheel in the build log (#1007).
+- Errors installing pip/setuptools/wheel are now displayed in the build output and fail the build early (#1007).
+- Install pip using itself rather than `get-pip.py` (#1007).
+- Disable pip's version check + cache when installing pip/setuptools/wheel (#1007).
+- Install setuptools from PyPI rather than a vendored copy (#1007).
+- Reduce the number of environment variables exposed to `bin/{pre,post}_compile` and other subprocesses (#1011).
+
+## v173 (2020-07-21)
+
+- Python 3.8.5 is now available (CPython).
+
+## v172 (2020-07-17)
+
+- Python 3.8.4 is now available (CPython).
+
+## v171 (2020-07-07)
+
+- Python 3.6.11 and 3.7.8 are now available (CPython).
+
+## v170 (2020-05-19)
 
 - Python 2.7.18, 3.5.9, 3.7.7 and 3.8.3 are now available (CPython).
-- PyPy 2.7 and 3.6, version 7.3.1 are now available (Note: PyPy support is in beta)
+- PyPy 2.7 and 3.6, version 7.3.1 are now available (Note: PyPy support is in beta).
-- Docs: Fix explanation of runtime.txt generation when using pipenv
+- Docs: Fix explanation of runtime.txt generation when using pipenv.
-- Bugfix: Correctly detect Python version when using a `python_version` of `3.8` in `Pipfile.lock`
+- Bugfix: Correctly detect Python version when using a `python_version` of `3.8` in `Pipfile.lock`.
 
---------------------------------------------------------------------------------
+## v169 (2020-04-22)
-
-# 169 (2020-04-22)
 
 - Add a Hatchet test for python 3.8.2
 - Set Code Owners to @heroku/langauges
 - Bugfix: Caching on subsequent redeploys
 - Update tests to support latest version of Python
 
-# 168 (2020-04-06)
+## v168 (2020-04-06)
 
 - Doc: Update Readme with version numbers
 - update Code Owners to include the Heroku Buildpack Maintainers team
-- Deprecation warning: BUILD_WITH_GEO_LIBRARIES is now deprecated. See warning for details.
+- Deprecation warning: `BUILD_WITH_GEO_LIBRARIES` is now deprecated. See warning for details.
 - Clean up build log output
 - Update Python versions in README to match docs
 - Django version detection fixed, link updated
 
-# 167 (2020-03-26)
+## v167 (2020-03-26)
 
 - Add failcase for cache busting
 - Bugfix: Clearing pip dependencies
 
-# 166 (2020-03-05)
+## v166 (2020-03-05)
 
 - Correct ftp to https in vendored file
 - Warn for Django 1.11 approaching EOL, provide link to roadmap
 
-# 165 (2020-02-27)
+## v165 (2020-02-27)
 
 - Python 3.8.2 now available.
 
-# 164 (2020-02-20)
+## v164 (2020-02-20)
 
 - Update requirements.txt builds to use Pip 20.0.2
 - Download get-pip.py to tmpdir instead of root dir
 
-# 163 (2019-12-23)
+## v163 (2019-12-23)
 
 - New pythons released:
   Python 3.8.1, 3.7.6, 3.6.10 (CPython)
   Beta Release: Pypy 2.7 and 3.6, version 7.2.0
 
-# 162 (2019-12-06)
+## v162 (2019-12-06)
 
 - Bug fix: fragile sqlite3 install
 
-# 161 (2019-12-2)
+## v161 (2019-12-2)
 
 - Bug fix: Sqlite3 version bump
 
-# 160 (2019-10-23)
+## v160 (2019-10-23)
 
 - Bugfix: Pipenv no longer installs twice in CI
 
-# 159 (2019-10-22)
+## v159 (2019-10-22)
 
 - Python 2.7.17 now available on Heroku 18 and 16.
 
-# 158 (2019-10-21)
+## v158 (2019-10-21)
 
 - Python 3.7.5 and 3.8.0 now available on Heroku 18 and 16.
 - Add support for Python 3.8 branch
@@ -74,11 +122,11 @@
   - Test Improvements
 - Add support for staging binary testing
 
-# 157 (2019-09-18)
+## v157 (2019-09-18)
 
 - Typo fixes
 
-# 156 (2019-09-12)
+## v156 (2019-09-12)
 
 - Python 3.6.9 and 3.7.4 now available.
 
@@ -86,125 +134,125 @@
 - Build utility and documentation updates
 - Bump Hatchet tests to point at new default python version.
 
-# 155 (2019-08-22)
+## v155 (2019-08-22)
 
 add docs and make target for heroku-18 bob builds
 
-# 154 (2019-07-17)
+## v154 (2019-07-17)
 
 Fix python 3.5.7 formula actually building 3.7.2
 
-# 153 (2019-06-21)
+## v153 (2019-06-21)
 
 Hotfix for broken heroku-16 deploys
 
-# 152 (2019-04-04)
+## v152 (2019-04-04)
 
 Python 3.7.3 now available.
 
-# 151 (2019-03-21)
+## v151 (2019-03-21)
 
 Python 3.5.7 and 3.4.10 now available on all Heroku stacks.
 
-# 150 (2019-03-13)
+## v150 (2019-03-13)
 
 Python 2.7.16 now available on all Heroku stacks.
 
-# 149 (2019-03-04)
+## v149 (2019-03-04)
 
 Hotfix for broken Cedar 14 deploys
 
-# 148 (2019-02-21)
+## v148 (2019-02-21)
 
 No user facing changes, improving internal metrics
 
-# 147 (2019-02-07)
+## v147 (2019-02-07)
 
 Python 3.7.2 and 3.6.8 now available on all Heroku stacks.
 
-# 146 (2018-11-11)
+## v146 (2018-11-11)
 
 Python 3.7.1, 3.6.7, 3.5.6 and 3.4.9 now available on all Heroku stacks.
 
-# 145 (2018-11-08)
+## v145 (2018-11-08)
 
 Testing and tooling expanded to better support new runtimes
 
-# 144 (2018-10-10)
+## v144 (2018-10-10)
 
 Switch to cautious upgrade for Pipenv install to ensure the pinned pip version
 is used with Pipenv
 
-# 143 (2018-10-09)
+## v143 (2018-10-09)
 
-Add support for detecting SLUGIFY_USES_TEXT_UNIDECODE, which is required to
+Add support for detecting `SLUGIFY_USES_TEXT_UNIDECODE`, which is required to
 install Apache Airflow version 1.10 or higher.
 
-# 142 (2018-10-08)
+## v142 (2018-10-08)
 
 Improvements to Python install messaging
 
-# 139, 140, 141
+## v139, 140, 141
 
 No user-facing changes, documenting for version clarity
 
-# 138 (2018-08-01)
+## v138 (2018-08-01)
 
 Use stack image SQLite3 instead of vendoring
 
-# 137 (2018-07-17)
+## v137 (2018-07-17)
 
 Prevent 3.7.0 from appearing as unsupported in buildpack messaging.
 
-# 136 (2018-06-28)
+## v136 (2018-06-28)
 
 Upgrade to 3.6.6 and support 3.7.0 on all runtimes.
 
-# 135 (2018-05-29)
+## v135 (2018-05-29)
 
 Upgrade Pipenv to v2018.5.18.
 
-# 134 (2018-05-02)
+## v134 (2018-05-02)
 
 Default to 3.6.5, bugfixes.
 
-# 133
+## v133
 
 Fixes for Pip 10 release.
 
-# 132
+## v132
 
 Improve pip installation, with the release of v9.0.2.
 
-# 131
+## v131
 
 Fix bug with pip.
 
-# 130
+## v130
 
 Better upgrade strategy for pip.
 
-# 129
+## v129
 
 Don't upgrade pip (from v128).
 
-# 128
+## v128
 
 Upgrade pip, pin to Pipenv v11.8.2.
 
-# 127
+## v127
 
 Pin to Pipenv v11.7.1.
 
-# 126
+## v126
 
 Bugfixes.
 
-# 125
+## v125
 
 Bugfixes.
 
-# 124
+## v124
 
 Update buildpack to automatically install `[dev-packages]` during Heroku CI Pipenv builds.
 
@@ -213,101 +261,101 @@ Update buildpack to automatically install `[dev-packages]` during Heroku CI Pipe
 - No longer warn if there is no `Procfile`.
 - Update Pipenv's "3.6" runtime specifier to point to "3.6.4".
 
-# 123
+## v123
 
 Update gunicorn `init.d` script to allow overrides.
 
-# 122
+## v122
 
 Update default Python to v3.6.4.
 
-# 121
+## v121
 
 Update default Python to v3.6.3.
 
-# 120
+## v120
 
 Use `$ pipenv --deploy`.
 
-# 119
+## v119
 
 Improvements to Pipenv support, warning on unsupported Python versions.
 
 - We now warn when a user is not using latest 2.x or 3.x Python.
 - Heroku now supports `[requires]` `python_full_version` in addition to `python_version`.
 
-# 118
+## v118
 
 Improvements to Pipenv support.
 
-# 117
+## v117
 
 Bug fix.
 
-# 116
+## v116
 
 Vendoring improvements.
 
 - Geos libraries should work on Heroku-16 now.
 - The libffi/libmemcached vendoring step is now skipped on Heroku-16 (since they are installed in the base image).
 
-# 115
+## v115
 
 Revert a pull request.
 
 - No longer using `sub_env` for `pip install` step.
 
-# 114
+## v114
 
 - Bugfixes.
 
 Blacklisting `PYTHONHOME` and `PYTHONPATH` for older apps. Upgrades to nltk support.
 
-# 113
+## v113
 
 Updates to Pipenv support.
 
-# 112
+## v112
 
 Bugfix.
 
 - Fixed grep output bug.
 
-# 111
+## v111
 
 Linting, bugfixes.
 
-# 110
+## v110
 
 Update default Python to 3.6.2.
 
-# 109
+## v109
 
 Update Default Python to 3.6.1, bugfixes.
 
 - Fixed automatic pip uninstall of dependencies removed from requirements.txt.
 
-# 108
+## v108
 
 Fix output for collectstatic step.
 
-# 107
+## v107
 
 Bugfix for C dependency installation.
 
-# 106
+## v106
 
- Don't install packages that could mess up packaging.
+Don't install packages that could mess up packaging.
 
  - The Python buildpack will automatically remove `six`, `pyparsing`, `appdirs`,
    `setuptools`, and `distribute` from a `requirements.txt` file now, as these
    packages are provided by the Python buildpack.
 
- # 105
+## v105
 
- Improvements to output messaging.
+Improvements to output messaging.
 
-# 104
+## v104
 
 General improvements.
 
@@ -317,52 +365,52 @@ General improvements.
 
 [395]: https://github.com/heroku/heroku-buildpack-python/pull/395
 
-## 103
+## v103
 
 Bug fixes and improvements.
 
 - Fix for Pipenv.
 - Fix for Heroku CI.
-- Improve handling of WEB_CONCURRENCY when using multiple buildpacks.
+- Improve handling of `WEB_CONCURRENCY` when using multiple buildpacks.
-- Adjust environment variables set during the build to more closely match those in the dyno environment (DYNO is now available, STACK is not).
+- Adjust environment variables set during the build to more closely match those in the dyno environment (`DYNO` is now available, `STACK` is not).
 - Restore the build cache prior to running bin/pre_compile.
 
-## 102
+## v102
 
 Buildpack code cleanup.
 
 - Improved messaging around NLTK.
 
-## 101
+## v101
 
 Updated setuptools installation method.
 
 - Improved pipenv support.
 
-## 100
+## v100
 
 Preliminary pipenv support.
 
-## 99
+## v99
 
 Cleanup.
 
-## 98
+## v98
 
 Official NLTK support and other improvements.
 
 - Support for `nltk.txt` file for declaring corpora to be downloaded.
-- Leading zeros for auto-set WEB_CONCURRENCY.
+- Leading zeros for auto-set `WEB_CONCURRENCY`.
 
-## 97
+## v97
 
 Improved egg-link functionality.
 
-## 96
+## v96
 
 Bugfix.
 
-## 95
+## v95
 
 Improved output support.
 
@@ -398,7 +446,7 @@ Fixed bug with editable pip installations.
 
 Updated default Python 2.7.13.
 
-- Python 2.7.13 uses UCS-4 build, more compatibile with linux wheels.
+- Python 2.7.13 uses UCS-4 build, more compatible with linux wheels.
 - Updated setuptools to v32.1.0.
 
 ## v86
@@ -424,7 +472,7 @@ Support for Heroku CI.
 
 ## v82 (2016-08-22)
 
-Update to library detection mechnisms (pip-pop).
+Update to library detection mechanisms (pip-pop).
 
 - Updated setuptools to v25.5.0
 
@@ -498,7 +546,7 @@ Revert to v66.
 
 ## v68 (2015-10-12)
 
-Fixed .heroku/venv error with modern apps.
+Fixed `.heroku/venv` error with modern apps.
 
 ## v67 (2015-10-12)
 
@@ -523,7 +571,6 @@ Updated Pip and Setuptools.
 - Setuptools updated to v18.3.2
 - Pip updated to v7.1.2
 
-
 ## v62 (2015-08-07)
 
 Updated Pip and Setuptools.

Gemfile.lock

@@ -1,68 +1,50 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.2.2)
+    diff-lcs (1.4.4)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2)
-    concurrent-ruby (1.1.6)
-    diff-lcs (1.3)
     erubis (2.7.0)
-    excon (0.73.0)
+    excon (0.76.0)
-    heroics (0.0.25)
+    heroics (0.1.1)
       erubis (~> 2.0)
       excon
       moneta
       multi_json (>= 1.9.2)
-    heroku_hatchet (5.0.3)
+    heroku_hatchet (7.1.1)
       excon (~> 0)
-      minitest-retry (~> 0.1.9)
+      platform-api (~> 3)
-      platform-api (~> 2)
-      repl_runner (~> 0.0.3)
       rrrretry (~> 1)
       thor (~> 0)
       threaded (~> 0)
-    i18n (1.8.2)
-      concurrent-ruby (~> 1.0)
-    minitest (5.14.0)
-    minitest-retry (0.1.9)
-      minitest (>= 5.0)
     moneta (1.0.0)
-    multi_json (1.14.1)
+    multi_json (1.15.0)
-    parallel (1.19.1)
+    parallel (1.19.2)
-    parallel_split_test (0.7.0)
+    parallel_split_test (0.8.0)
       parallel (>= 0.5.13)
       rspec (>= 3.1.0)
-    platform-api (2.2.0)
+    platform-api (3.0.0)
-      heroics (~> 0.0.25)
+      heroics (~> 0.1.1)
       moneta (~> 1.0.0)
-    rake (12.3.1)
+      rate_throttle_client (~> 0.1.0)
-    repl_runner (0.0.3)
+    rake (13.0.1)
-      activesupport
+    rate_throttle_client (0.1.2)
     rrrretry (1.0.0)
-    rspec (3.8.0)
+    rspec (3.9.0)
-      rspec-core (~> 3.8.0)
+      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.8.0)
+      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.8.0)
+      rspec-mocks (~> 3.9.0)
-    rspec-core (3.8.0)
+    rspec-core (3.9.2)
-      rspec-support (~> 3.8.0)
+      rspec-support (~> 3.9.3)
-    rspec-expectations (3.8.1)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
+      rspec-support (~> 3.9.0)
-    rspec-mocks (3.8.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
+      rspec-support (~> 3.9.0)
-    rspec-retry (0.6.1)
+    rspec-retry (0.6.2)
       rspec-core (> 3.3)
-    rspec-support (3.8.0)
+    rspec-support (3.9.3)
     thor (0.20.3)
-    thread_safe (0.3.6)
     threaded (0.0.4)
-    tzinfo (1.2.7)
-      thread_safe (~> 0.1)
-    zeitwerk (2.3.0)
 
 PLATFORMS
   ruby

Makefile

@@ -1,26 +1,26 @@
 # These targets are not files
-.PHONY: tests
+.PHONY: check test buildenv-heroku-16 buildenv-heroku-18 tools
 
-test: test-heroku-18 test-heroku-16 test-cedar-14
+STACK ?= heroku-18
+TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
+
+ifeq ($(STACK),cedar-14)
+	# Cedar-14 doesn't have a build image varient.
+	IMAGE_TAG := heroku/cedar:14
+else
+	# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
+	IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
+endif
 
 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
 	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
 	@shellcheck -x bin/steps/hooks/*
 
-test-cedar-14:
+test:
-	@echo "Running tests in docker (cedar-14)..."
+	@echo "Running tests using: STACK=$(STACK) TEST_CMD='$(TEST_CMD)'"
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
-
+	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
-test-heroku-16:
-	@echo "Running tests in docker (heroku-16)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
-	@echo ""
-
-test-heroku-18:
-	@echo "Running tests in docker (heroku-18)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
 
 buildenv-heroku-16:

NOTICE

@@ -86,31 +86,6 @@ Each version is given a distinguishing version number. If the Library as you rec
 
 If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library.
 
-get-pip.py license
-------------------
-
-Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-
 pip-pop license
 ---------------
 
@@ -134,4 +109,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+THE SOFTWARE.

README.md

@@ -2,7 +2,7 @@
 
 # Heroku Buildpack: Python
 
-[![Build Status](https://travis-ci.org/heroku/heroku-buildpack-python.svg?branch=master)](https://travis-ci.org/heroku/heroku-buildpack-python)
+[![Build Status](https://travis-ci.com/heroku/heroku-buildpack-python.svg?branch=main)](https://travis-ci.com/heroku/heroku-buildpack-python)
 
 This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/buildpacks) for Python apps.
 
@@ -16,7 +16,7 @@ See it in Action
 $ ls
 my-application		requirements.txt	runtime.txt
 
-$ git push heroku master
+$ git push heroku main
 Counting objects: 4, done.
 Delta compression using up to 8 threads.
 Compressing objects: 100% (2/2), done.
@@ -26,7 +26,7 @@ remote: Compressing source files... done.
 remote: Building source:
 remote:
 remote: -----> Python app detected
-remote: -----> Installing python-3.7.4
+remote: -----> Installing python
 remote: -----> Installing pip
 remote: -----> Installing SQLite3
 remote: -----> Installing requirements with pip
@@ -44,7 +44,7 @@ A `requirements.txt` must be present at the root of your application's repositor
 
 To specify your python version, you also need a `runtime.txt` file - unless you are using the default Python runtime version.
 
-Current default Python Runtime: Python 3.6.9
+Current default Python Runtime: Python 3.6.12
 
 Alternatively, you can provide a `setup.py` file, or a `Pipfile`.
 Using `pipenv` will generate `runtime.txt` at build time if one of the field `python_version` or `python_full_version` is specified in the `requires` section of your `Pipfile`.
@@ -62,9 +62,9 @@ Specify a Python Runtime
 
 Supported runtime options include:
 
-- `python-3.8.3`
+- `python-3.8.5`
-- `python-3.7.7`
+- `python-3.7.9`
-- `python-3.6.10`
+- `python-3.6.12`
 - `python-2.7.18`
 
 ## Tests
@@ -72,17 +72,22 @@ Supported runtime options include:
 The buildpack tests use [Docker](https://www.docker.com/) to simulate
 Heroku's [stack images.](https://devcenter.heroku.com/articles/stack)
 
-To run the test suite:
+To run the test suite against the default stack:
 
 ```
 make test
 ```
 
-Or to test in a particular stack:
+Or to test against a particular stack:
 
 ```
-make test-heroku-18
+make test STACK=heroku-16
-make test-heroku-16
+```
+
+To run only a subset of the tests:
+
+```
+make test TEST_CMD=tests/versions
 ```
 
 The tests are run via the vendored

bin/compile

@@ -16,7 +16,9 @@
 set -eo pipefail
 
 # Boostrap the Buildpack Standard Library.
-export BPLOG_PREFIX="buildpack.python"
+# Disable unused env var warning since shellcheck doesn't know about the stdlib.
+# shellcheck disable=2034
+BPLOG_PREFIX="buildpack.python"
 export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
 
 [ "$BUILDPACK_XTRACE" ] && set -o xtrace
@@ -63,37 +65,13 @@ PYPY36="pypy3.6"
 
 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
-# If pip doesn't match this version (the version we install), run the installer.
-PIP_UPDATE="20.0.2"
-
-for file in "$BUILD_DIR/runtime.txt" "$CACHE_DIR/.heroku/python-version" ; do
-    [ -f "$file" ] || continue
-
-    version=$(tr -d '[:space:]' < "$file")
-
-    case "$version" in "$PY34"*)
-        # Python 3.4 support was dropped in pip >= 19.2.
-        PIP_UPDATE="19.1.1"
-        break
-        ;;
-    esac
-done
-
-if [[ -f "$BUILD_DIR/Pipfile" ]]; then
-    # Do not force pipenv users to re-install pipenv locally.
-    PIP_UPDATE="9.0.2"
-fi
-
-export DEFAULT_PYTHON_STACK PIP_UPDATE
-export PY37 PY36 PY35 PY27 PY34
 
 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.
 # The `warnings` subscript then greps through this for common problems and guides
 # the user towards resolution of known issues.
 WARNINGS_LOG=$(mktemp)
-export WARNINGS_LOG
+RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
-export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
 
 # The buildpack ships with a few executable tools (e.g. pip-grep, etc).
 # This installs them into the path, so we can execute them directly.
@@ -159,6 +137,10 @@ export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PAT
 export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
 export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
 
+# Global pip options (https://pip.pypa.io/en/stable/user_guide/#environment-variables).
+# Disable pip's warnings about EOL Python since we show our own.
+export PIP_NO_PYTHON_VERSION_WARNING=1
+
 # The Application Code
 # --------------------
 
@@ -217,8 +199,6 @@ else
   CACHED_PYTHON_STACK=$STACK
 fi
 
-export CACHED_PYTHON_STACK
-
 # Pipenv Python version support.
 # Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version).
 # Convert it to a runtime.txt file.

bin/default_pythons

@@ -1,21 +1,16 @@
 #!/usr/bin/env bash
 
-DEFAULT_PYTHON_VERSION="python-3.6.10"
+# Disable unused env var warning, since shellcheck doesn't take into account
-LATEST_38="python-3.8.3"
+# that this file is sourced. We don't want to use export since it exposes
-LATEST_37="python-3.7.7"
+# the env vars to subprocesses.
-LATEST_36="python-3.6.10"
+# shellcheck disable=2034
-LATEST_35="python-3.5.9"
+
+DEFAULT_PYTHON_VERSION="python-3.6.12"
+LATEST_38="python-3.8.5"
+LATEST_37="python-3.7.9"
+LATEST_36="python-3.6.12"
+LATEST_35="python-3.5.10"
 LATEST_34="python-3.4.10"
 LATEST_27="python-2.7.18"
-PYPY_36="pypy3.6-7.3.1"
+LATEST_PYPY_36="pypy3.6-7.3.1"
-PYPY_27="pypy2.7-7.3.1"
+LATEST_PYPY_27="pypy2.7-7.3.1"
-
-export DEFAULT_PYTHON_VERSION \
-    LATEST_38 \
-    LATEST_37 \
-    LATEST_36 \
-    LATEST_35 \
-    LATEST_34 \
-    LATEST_27 \
-    PYPY_36 \
-    PYPY_27

bin/steps/README.MD

@@ -2,13 +2,6 @@
 
 TODO: Add context on Python install steps, such as why symlinking vs copying
 
-## Installing the Pip tool
-
-The Python Buildpack uses a tool called `get-pip` to install the pip tool. This
-is done in the `python` script.
-
-This is in part because Python historically did not come with pip by default.
-
 ## Installing Python packages using Pip
 
 ### Convention: Use `python` process to invoke Pip

bin/steps/python

@@ -8,6 +8,7 @@ PYTHON_VERSION=$(cat runtime.txt)
 VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"
 
 SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
+SECURITY_UPDATE_PYPY="The PyPy project has released a security update! Please consider upgrading to"
 
 ONLY_SUPPORTED_2_VERSION="Only the latest version of Python 2 is supported on the platform. Please consider upgrading to"
 
@@ -51,11 +52,9 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
     fi
   fi
   if [[ "$PYTHON_VERSION" == $PY27* ]]; then
+    puts-warn "$PYTHON_2_EOL_UPDATE"
+    echo "       Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq"
     # security update note
-    if [[ "$(date "+%Y")" -gt "2019" ]]; then
-      puts-warn "$PYTHON_2_EOL_UPDATE"
-      echo "       Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq"
-    fi
     if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then
       puts-warn "$ONLY_SUPPORTED_2_VERSION" "$LATEST_27"
       echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
@@ -63,15 +62,15 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
   fi
   if [[ "$PYTHON_VERSION" == $PYPY27* ]]; then
     # security update note
-    if [ "$PYTHON_VERSION" != "$PYPY_27" ]; then
+    if [ "$PYTHON_VERSION" != "$LATEST_PYPY_27" ]; then
-      puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_27}?"
+      puts-warn "$SECURITY_UPDATE_PYPY" "$LATEST_PYPY_27"
       echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
     fi
   fi
   if [[ "$PYTHON_VERSION" == $PYPY36* ]]; then
     # security update note
-    if [ "$PYTHON_VERSION" != "$PYPY_36" ]; then
+    if [ "$PYTHON_VERSION" != "$LATEST_PYPY_36" ]; then
-      puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_36}?"
+      puts-warn "$SECURITY_UPDATE_PYPY" "$LATEST_PYPY_36"
       echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
     fi
   fi
@@ -131,34 +130,58 @@ if [ ! "$SKIP_INSTALL" ]; then
   # Record for future reference.
   echo "$PYTHON_VERSION" > .heroku/python-version
   echo "$STACK" > .heroku/python-stack
-  FRESH_PYTHON=true
 
   hash -r
 fi
 
-# Heroku uses the get-pip utility maintained by the Python community to vendor Pip.
-# https://github.com/pypa/get-pip
-GETPIP_URL="https://lang-python.s3.amazonaws.com/etc/get-pip.py"
-GETPIP_PY="${TMPDIR:-/tmp}/get-pip.py"
-
-if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then
-    mcount "failure.python.get-pip"
-    echo "Failed to pull down get-pip"
-    exit 1
-fi
-
-# If a new Python has been installed or Pip isn't up to date:
-if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
-
-  puts-step "Installing pip"
-
-  # Remove old installations.
-  rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
-  rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
-
-  /app/.heroku/python/bin/python "$GETPIP_PY" pip=="$PIP_UPDATE" &> /dev/null
-  /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
-fi
-
 set -e
+
+PIP_VERSION='20.1.1'
+# Must use setuptools <47.2.0 until we fix:
+# https://github.com/heroku/heroku-buildpack-python/issues/1006
+SETUPTOOLS_VERSION='47.1.1'
+WHEEL_VERSION='0.34.2'
+
+if [[ "${PYTHON_VERSION}" == ${PY34}* ]]; then
+  # Python 3.4 support was dropped in pip 19.2+, setuptools 44+ and wheel 0.34+.
+  PIP_VERSION='19.1.1'
+  SETUPTOOLS_VERSION='43.0.0'
+  WHEEL_VERSION='0.33.6'
+elif [[ "${PYTHON_VERSION}" == ${PY27}* || "${PYTHON_VERSION}" == ${PYPY27}* ]]; then
+  # Python 2.7 support was dropped in setuptools 45+.
+  SETUPTOOLS_VERSION='44.1.1'
+fi
+
+# We don't use get-pip.py, since:
+#  - it uses `--force-reinstall`, which is unnecessary here and slows down repeat builds
+#  - it means downloading pip twice (once embedded in get-pip.py, and again during
+#    the install, since get-pip.py can't install the embedded version directly)
+#  - we would still have to manage several versions of get-pip.py, to support older Pythons.
+# Instead, we use the pip wheel to install itself, using the method described here:
+# https://github.com/pypa/pip/issues/2351#issuecomment-69994524
+PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl"
+PIP_WHEEL_URL="https://lang-python.s3.amazonaws.com/common/${PIP_WHEEL_FILENAME}"
+PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}"
+
+if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then
+  mcount "failure.python.download-pip"
+  puts-warn "Failed to download pip"
+  exit 1
+fi
+
+if [[ -f "$BUILD_DIR/Pipfile" ]]; then
+  # The buildpack is pinned to old pipenv, which requires older pip.
+  # Pip 9.0.2 doesn't support installing itself from a wheel, so we have to use split
+  # versions here (ie: installer pip version different from target pip version).
+  PIP_VERSION='9.0.2'
+  PIP_TO_INSTALL="pip==${PIP_VERSION}"
+else
+  PIP_TO_INSTALL="${PIP_WHEEL}"
+fi
+
+puts-step "Installing pip ${PIP_VERSION}, setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}"
+
+/app/.heroku/python/bin/python "${PIP_WHEEL}/pip" install --quiet --disable-pip-version-check --no-cache \
+  "${PIP_TO_INSTALL}" "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}"
+
 hash -r

builds/runtimes/python-2.7.17

@@ -1,27 +1,4 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
 
-OUT_PREFIX=$1
+source $(dirname $0)/python2
-BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
-export BIN_DIR
-
-# shellcheck source=bin/utils
-source "$BIN_DIR/steps/sqlite3"
-
-sqlite3_version
-echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
-sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
-
-echo "Building Python…"
-SOURCE_TARBALL='https://python.org/ftp/python/2.7.17/Python-2.7.17.tgz'
-curl -L $SOURCE_TARBALL | tar xz
-mv Python-2.7.17 src
-cd src
-
-./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
-make
-make install
-
-# Remove unneeded test directories, similar to the official Docker Python images:
-# https://github.com/docker-library/python
-find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +

builds/runtimes/python-2.7.18

@@ -1,27 +1,4 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
 
-OUT_PREFIX=$1
+source $(dirname $0)/python2
-BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
-export BIN_DIR
-
-# shellcheck source=bin/utils
-source "$BIN_DIR/steps/sqlite3"
-
-sqlite3_version
-echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
-sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
-
-echo "Building Python…"
-SOURCE_TARBALL='https://python.org/ftp/python/2.7.18/Python-2.7.18.tgz'
-curl -L $SOURCE_TARBALL | tar xz
-mv Python-2.7.18 src
-cd src
-
-./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
-make
-make install
-
-# Remove unneeded test directories, similar to the official Docker Python images:
-# https://github.com/docker-library/python
-find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +

builds/runtimes/python-3.4.10

@@ -1,32 +1,4 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
 
-OUT_PREFIX=$1
+source $(dirname $0)/python3
-BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
-export BIN_DIR
-
-# shellcheck source=bin/utils
-source "$BIN_DIR/steps/sqlite3"
-
-sqlite3_version
-echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
-sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
-
-echo "Building Python…"
-SOURCE_TARBALL='https://python.org/ftp/python/3.7.2/Python-3.7.2.tgz'
-curl -L $SOURCE_TARBALL | tar xz
-mv Python-3.7.2 src
-cd src
-
-./configure --prefix=$OUT_PREFIX --with-ensurepip=no
-make
-make install
-
-# Remove unneeded test directories, similar to the official Docker Python images:
-# https://github.com/docker-library/python
-find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
-
-# Remove spare /
-LOCATION=${OUT_PREFIX%?}
-
-ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.4.9

@@ -1,32 +1,4 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
 
-OUT_PREFIX=$1
+source $(dirname $0)/python3
-BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
-export BIN_DIR
-
-# shellcheck source=bin/utils
-source "$BIN_DIR/steps/sqlite3"
-
-sqlite3_version
-echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
-sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
-
-echo "Building Python…"
-SOURCE_TARBALL='https://python.org/ftp/python/3.4.9/Python-3.4.9.tgz'
-curl -L $SOURCE_TARBALL | tar xz
-mv Python-3.4.9 src
-cd src
-
-./configure --prefix=$OUT_PREFIX --with-ensurepip=no
-make
-make install
-
-# Remove unneeded test directories, similar to the official Docker Python images:
-# https://github.com/docker-library/python
-find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
-
-# Remove spare /
-LOCATION=${OUT_PREFIX%?}
-
-ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.5.10

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.6.11

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.6.12

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.7.8

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.7.9

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.4

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.5

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

etc/ci-setup.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-sudo apt-get -qq update
-sudo apt-get install software-properties-common
-curl --fail --retry 3 --retry-delay 1 --connect-timeout 3 --max-time 30 https://cli-assets.heroku.com/install-ubuntu.sh | sh

etc/publish.sh

@@ -14,9 +14,9 @@ case "$choice" in
   * ) exit 1;;
 esac
 
-originMaster=$(git rev-parse origin/master)
+originMain=$(git rev-parse origin/main)
-echo "Tagging commit $originMaster with $newVersion... "
+echo "Tagging commit $originMain with $newVersion... "
-git tag "$newVersion" "${originMaster:?}"
+git tag "$newVersion" "${originMain:?}"
 git push origin refs/tags/$newVersion
 
 heroku buildpacks:publish "$BP_NAME" "$newVersion"
@@ -31,7 +31,7 @@ if [ $(git tag | grep -q latest-version) ]; then
     echo "Updating latest-version tag"
     git tag -d latest-version
     git push origin :latest-version
-    git tag latest-version "${originMaster:?}"
+    git tag latest-version "${originMain:?}"
     git push --tags
 fi
 

requirements.txt

@@ -1,3 +1,3 @@
 docopt==0.6.2
-bob-builder==0.0.17
+bob-builder==0.0.18
-boto==2.48.0
+boto==2.49.0

test/fixtures/hooks/bin/post_compile

@@ -0,0 +1,4 @@
+set -euo pipefail
+
+echo "post_compile ran!"
+echo "post_compile env: $(printenv | cut -d '=' -f 1 | sort | xargs)."

test/fixtures/hooks/bin/pre_compile

@@ -0,0 +1,4 @@
+set -euo pipefail
+
+echo "pre_compile ran!"
+echo "pre_compile env: $(printenv | cut -d '=' -f 1 | sort | xargs)."

test/fixtures/pypy2_7/runtime.txt

@@ -1 +1 @@
-pypy2.7-7.2.0
+pypy2.7-7.3.1

test/fixtures/pypy2_7_warn/runtime.txt

@@ -0,0 +1 @@
+pypy2.7-7.2.0

test/fixtures/pypy3_6/runtime.txt

@@ -1 +1 @@
-pypy3.6-7.2.0
+pypy3.6-7.3.1

test/fixtures/pypy3_6_warn/runtime.txt

@@ -0,0 +1 @@
+pypy3.6-7.2.0

test/fixtures/python3_5/runtime.txt

@@ -1 +1 @@
-python-3.5.9
+python-3.5.10

test/run-deps

@@ -5,8 +5,9 @@
 source "bin/default_pythons"
 
 testAirflow() {
-  export SLUGIFY_USES_TEXT_UNIDECODE="yes"
+  local env_dir="$(mktmpdir)"
-  compile "airflow"
+  echo 'yes' > "${env_dir}/SLUGIFY_USES_TEXT_UNIDECODE"
+  compile 'airflow' '' "${env_dir}"
   assertCaptured "apache-airflow==1.10.2"
   assertCapturedSuccess
 }
@@ -17,27 +18,30 @@ testCollectstatic() {
 }
 
 testGEOS() {
-  export BUILD_WITH_GEO_LIBRARIES=1
+  local env_dir="$(mktmpdir)"
-  compile "geos"
+  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
+  compile 'geos' '' "${env_dir}"
   assertCaptured "geos"
   assertCapturedSuccess
 }
 
 testGEOSDeprecation() {
-  export BUILD_WITH_GEO_LIBRARIES=1
+  local env_dir="$(mktmpdir)"
-  compile "geos"
+  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
+  compile 'geos' '' "${env_dir}"
   assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated.
  !     An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
   assertCapturedSuccess
 }
 
 testNLTK() {
+  local env_dir="$(mktmpdir)"
   # NOTE: This is a RuntimeWarning emitted by Python 3's runpy.py script
   # which is what is used when you call `python -m <module>`. This is due to
   # how nltk imports things. It's not actually an error, but it would probably
   # be bad to silence in Production.
-  export PYTHONWARNINGS="ignore::RuntimeWarning"
+  echo 'ignore::RuntimeWarning' > "${env_dir}/PYTHONWARNINGS"
-  compile "nltk"
+  compile 'nltk' '' "${env_dir}"
   assertCaptured "[nltk_data] Downloading package city_database" "STD_ERR"
   assertCapturedSuccess
 }

test/run-features

@@ -44,6 +44,7 @@ testStandardRequirements() {
 
 testPipenv() {
   compile "pipenv"
+  assertCaptured "Installing pip 9.0.2, setuptools 47.1.1 and wheel 0.34.2"
   assertCapturedSuccess
 }
 
@@ -86,6 +87,50 @@ testDontWarnOldDjango() {
   assertCapturedSuccess
 }
 
+testHooks() {
+  # Test that the hooks are called correctly, and that the environment contains
+  # the app's config vars but no unexpected env vars from the buildpack.
+  local env_dir="$(mktmpdir)"
+  echo 'test' > "${env_dir}/SOME_APP_CONFIG_VAR"
+  local expected_env_vars=(
+    _
+    BIN_DIR
+    BUILD_DIR
+    BUILDPACK_LOG_FILE
+    CACHE_DIR
+    C_INCLUDE_PATH
+    CPLUS_INCLUDE_PATH
+    ENV_DIR
+    EXPORT_PATH
+    HOME
+    LANG
+    LD_LIBRARY_PATH
+    LIBRARY_PATH
+    OLDPWD
+    PATH
+    PIP_NO_PYTHON_VERSION_WARNING
+    PKG_CONFIG_PATH
+    PROFILE_PATH
+    PWD
+    PYTHONUNBUFFERED
+    SHLVL
+    SOME_APP_CONFIG_VAR
+    STACK
+    VENDOR_URL
+  )
+  if [[ "${STACK}" == "cedar-14" || "${STACK}" == "heroku-16" ]]; then
+    # Remove "OLDPWD" from expected_env_vars since for bash <4.4 it's not exported to subshells:
+    # https://github.com/heroku/heroku-buildpack-python/pull/1011#issuecomment-665117835
+    read -ra expected_env_vars <<< "${expected_env_vars[@]/OLDPWD/}"
+  fi
+  compile 'hooks' '' "${env_dir}"
+  assertCaptured "pre_compile ran!"
+  assertCaptured "pre_compile env: ${expected_env_vars[*]}."
+  assertCaptured "post_compile ran!"
+  assertCaptured "post_compile env: ${expected_env_vars[*]}."
+  assertCapturedSuccess
+}
+
 pushd $(dirname 0) >/dev/null
 popd >/dev/null
 

test/run-versions

@@ -9,6 +9,7 @@ testPythonDefault() {
   compile "pythonDefault"
   assertCaptured $DEFAULT_PYTHON_VERSION
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -18,12 +19,9 @@ testPython2() {
     echo $LATEST_27 > "runtime.txt"
     compile "python2"
     assertCaptured $LATEST_27
-    if [[ $(date "+%Y") > "2019" ]]; then
+    assertCaptured "python-2-7-eol-faq";
-      assertCaptured "python-2-7-eol-faq";
-    else
-      assertNotCaptured "python-2-7-eol-faq";
-    fi
     assertNotCaptured "security update"
+    assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
 }
@@ -31,11 +29,7 @@ testPython2() {
 testPython2_warn() {
     compile "python2_warn"
     assertCaptured "python-2.7.15"
-    if [[ $(date "+%Y") > "2019" ]]; then
+    assertCaptured "python-2-7-eol-faq";
-      assertCaptured "python-2-7-eol-faq";
-    else
-      assertNotCaptured "python-2-7-eol-faq";
-    fi
     assertCaptured "Only the latest version"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
@@ -51,29 +45,15 @@ testPython3_4() {
   compile "python3_4"
   assertCaptured $LATEST_34
   assertNotCaptured "security update"
-  # if cedar 14 and legacy binaries, fail. if cedar 14 and staging, succeed.
+  assertCaptured "Installing pip 19.1.1, setuptools 43.0.0 and wheel 0.33.6"
-  if [[ ! -n $USE_STAGING_BINARIES ]] && [[ $STACK == "cedar-14" ]]; then
+  assertCapturedSuccess
-      assertCapturedError
-# if heroku 18 and legacy binaries, succeed. if heroku 18 and staging, fail.
-  elif [[ -n $USE_STAGING_BINARIES ]] && [[ $STACK == "heroku-18" ]]; then
-      assertCapturedError
-  else
-#   all else succeed
-    assertCapturedSuccess
-  fi
 }
 
 testPython3_4_warn() {
   compile "python3_4_warn"
   assertCaptured "python-3.4.9"
   assertCaptured "security update!"
-# if heroku 18 and legacy binaries, succeed. if heroku 18 and staging, fail.
+  assertCapturedSuccess
-  if [[ -n $USE_STAGING_BINARIES ]] && [[ $STACK == "heroku-18" ]]; then
-      assertCapturedError
-  else
-#   all else succeed
-    assertCapturedSuccess
-  fi
 }
 
 testPython3_4_fail() {
@@ -86,6 +66,7 @@ testPython3_5() {
   compile "python3_5"
   assertCaptured $LATEST_35
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -108,6 +89,7 @@ testPython3_6() {
   compile "python3_6"
   assertCaptured $LATEST_36
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -134,6 +116,7 @@ testPython3_7() {
   else
     assertNotCaptured "security update"
     assertCaptured $LATEST_37
+    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
   fi
@@ -177,6 +160,7 @@ testPython3_8() {
   else
     assertNotCaptured "security update"
     assertCaptured $LATEST_38
+    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
   fi
@@ -190,22 +174,42 @@ testPython3_8_fail() {
 
 testPypy3_6() {
   compile "pypy3_6"
+  assertCaptured "Installing pypy"
+  assertNotCaptured "security update"
+  assertCaptured "$LATEST_PYPY_36"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCapturedSuccess
+}
+
+testPypy3_6_warn() {
+  compile "pypy3_6_warn"
   if [[ $STACK = "cedar-14" ]]; then
     assertCapturedError
   else
     assertCaptured "Installing pypy"
-    assertCaptured "$PYPY_36"
+    assertCaptured "security update!"
+    assertCaptured "$LATEST_PYPY_36"
     assertCapturedSuccess
   fi
 }
 
 testPypy2_7() {
   compile "pypy2_7"
+  assertCaptured "Installing pypy"
+  assertNotCaptured "security update"
+  assertCaptured "$LATEST_PYPY_27"
+  assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
+  assertCapturedSuccess
+}
+
+testPypy2_7_warn() {
+  compile "pypy2_7_warn"
   if [[ $STACK = "cedar-14" ]]; then
     assertCapturedError
   else
     assertCaptured "Installing pypy"
-    assertCaptured "$PYPY_27"
+    assertCaptured "security update!"
+    assertCaptured "$LATEST_PYPY_27"
     assertCapturedSuccess
   fi
 }

test/utils

@@ -56,21 +56,6 @@ resetCapture()
   unset rtrn # deprecated
 }
 
-detect()
-{
-  capture ${BUILDPACK_HOME}/bin/detect ${BUILD_DIR}
-}
-
-compile()
-{
-  capture ${BUILDPACK_HOME}/bin/compile ${BUILD_DIR} ${CACHE_DIR}
-}
-
-release()
-{
-  capture ${BUILDPACK_HOME}/bin/release ${BUILD_DIR}
-}
-
 updateVersion()
 {
   echo "$2" > "test/fixtures/${1}/runtime.txt"
@@ -234,13 +219,18 @@ default_process_types_cleanup() {
   fi
 }
 
+run_in_clean_env() {
+  # Prevent stray environment variables from outside the test runner being exposed to tests.
+  env -i HOME="${HOME}" LANG="${LANG}" PATH="${PATH}" STACK="${STACK}" "$@"
+}
+
 compile() {
   default_process_types_cleanup
   bp_dir=$(mktmpdir)
   compile_dir=$(mktmpdir)
   cp -a $(pwd)/* ${bp_dir}
   cp -a ${bp_dir}/test/fixtures/$1/. ${compile_dir}
-  capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3
+  capture run_in_clean_env ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3
 }
 
 compileDir() {
@@ -252,13 +242,13 @@ compileDir() {
   local env_dir=$3
 
   cp -a $(pwd)/* ${bp_dir}
-  capture ${bp_dir}/bin/compile ${compile_dir} ${cache_dir} ${env_dir}
+  capture run_in_clean_env ${bp_dir}/bin/compile ${compile_dir} ${cache_dir} ${env_dir}
 }
 
 release() {
   bp_dir=$(mktmpdir)
   cp -a $(pwd)/* ${bp_dir}
-  capture ${bp_dir}/bin/release ${bp_dir}/test/fixtures/$1
+  capture run_in_clean_env ${bp_dir}/bin/release ${bp_dir}/test/fixtures/$1
 }
 
 assertFile() {

tests.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-if [[ ! "$STACK" ]]; then
-    echo '$STACK must be set! (heroku-16 | cedar-14)'
-    exit 1
-fi
-
-if [[ "$STACK" == "cedar-14" ]]; then
-    make test-cedar-14
-    exit $?
-fi
-
-if [[ "$STACK" == "heroku-16" ]]; then
-    make test-heroku-16
-    exit $?
-fi
-
-if [[ "$STACK" == "heroku-18" ]]; then
-    make test-heroku-18
-    exit $?
-fi

vendor/test-utils

@@ -1,204 +0,0 @@
-#!/bin/sh
-
-# taken from
-# https://github.com/ryanbrainard/heroku-buildpack-testrunner/blob/master/lib/test_utils.sh
-
-oneTimeSetUp()
-{
-   TEST_SUITE_CACHE="$(mktemp -d ${SHUNIT_TMPDIR}/test_suite_cache.XXXX)"
-}
-
-oneTimeTearDown()
-{
-  rm -rf ${TEST_SUITE_CACHE}
-}
-
-setUp()
-{
-  OUTPUT_DIR="$(mktemp -d ${SHUNIT_TMPDIR}/output.XXXX)"
-  STD_OUT="${OUTPUT_DIR}/stdout"
-  STD_ERR="${OUTPUT_DIR}/stderr"
-  BUILD_DIR="${OUTPUT_DIR}/build"
-  CACHE_DIR="${OUTPUT_DIR}/cache"
-  mkdir -p ${OUTPUT_DIR}
-  mkdir -p ${BUILD_DIR}
-  mkdir -p ${CACHE_DIR}
-}
-
-tearDown()
-{
-  rm -rf ${OUTPUT_DIR}
-}
-
-capture()
-{
-  resetCapture
-
-  LAST_COMMAND="$@"
-
-  "$@" >${STD_OUT} 2>${STD_ERR}
-  RETURN=$?
-  rtrn=${RETURN} # deprecated
-}
-
-resetCapture()
-{
-  if [ -f ${STD_OUT} ]; then
-    rm ${STD_OUT}
-  fi
-
-  if [ -f ${STD_ERR} ]; then
-    rm ${STD_ERR}
-  fi
-
-  unset LAST_COMMAND
-  unset RETURN
-  unset rtrn # deprecated
-}
-
-detect()
-{
-  capture ${BUILDPACK_HOME}/bin/detect ${BUILD_DIR}
-}
-
-compile()
-{
-  capture ${BUILDPACK_HOME}/bin/compile ${BUILD_DIR} ${CACHE_DIR}
-}
-
-release()
-{
-  capture ${BUILDPACK_HOME}/bin/release ${BUILD_DIR}
-}
-
-assertCapturedEquals()
-{
-  assertEquals "$@" "$(cat ${STD_OUT})"
-}
-
-assertCapturedNotEquals()
-{
-  assertNotEquals "$@" "$(cat ${STD_OUT})"
-}
-
-assertCaptured()
-{
-  assertFileContains "$@" "${STD_OUT}"
-}
-
-assertNotCaptured()
-{
-  assertFileNotContains "$@" "${STD_OUT}"
-}
-
-assertCapturedSuccess()
-{
-  assertEquals "Expected captured exit code to be 0; was <${RETURN}>" "0" "${RETURN}"
-  assertEquals "Expected STD_ERR to be empty; was <$(cat ${STD_ERR})>" "" "$(cat ${STD_ERR})"
-}
-
-# assertCapturedError [[expectedErrorCode] expectedErrorMsg]
-assertCapturedError()
-{
-  if [ $# -gt 1 ]; then
-    expectedErrorCode=${1}
-    shift
-  fi
-
-  expectedErrorMsg=${1:-""}
-
-  if [ -z ${expectedErrorCode} ]; then
-    assertTrue "Expected captured exit code to be greater than 0; was <${RETURN}>" "[ ${RETURN} -gt 0 ]"
-  else
-    assertTrue "Expected captured exit code to be <${expectedErrorCode}>; was <${RETURN}>" "[ ${RETURN} -eq ${expectedErrorCode} ]"
-  fi
-
-  assertFileContains "Expected STD_OUT to contain error <${expectedErrorMsg}>" "${expectedErrorMsg}" "${STD_OUT}"
-  assertEquals       "STD_ERR should always be empty" "" "$(cat ${STD_ERR})"
-}
-
-assertAppDetected()
-{
-  expectedAppType=${1?"Must provide app type"}
-
-  assertCapturedSuccess
-  assertEquals "${expectedAppType}" "$(cat ${STD_OUT})"
-}
-
-assertNoAppDetected()
-{
-  assertEquals "1" "${RETURN}"
-  assertEquals "no" "$(cat ${STD_OUT})"
-  assertEquals "" "$(cat ${STD_ERR})"
-}
-
-_assertContains()
-{
-  if [ 5 -eq $# ]; then
-    msg=$1
-    shift
-  elif [ ! 4 -eq $# ]; then
-    fail "Expected 4 or 5 parameters; Receieved $# parameters"
-  fi
-
-  needle=$1
-  haystack=$2
-  expectation=$3
-  haystack_type=$4
-
-  case "${haystack_type}" in
-    "file") grep -q -F -e "${needle}" ${haystack} ;;
-    "text") echo "${haystack}" | grep -q -F -e "${needle}" ;;
-  esac
-
-  if [ "${expectation}" != "$?" ]; then
-    case "${expectation}" in
-      0) default_msg="Expected <${haystack}> to contain <${needle}>" ;;
-      1) default_msg="Did not expect <${haystack}> to contain <${needle}>" ;;
-    esac
-
-    fail "${msg:-${default_msg}}"
-  fi
-}
-
-assertContains()
-{
-  _assertContains "$@" 0 "text"
-}
-
-assertNotContains()
-{
-  _assertContains "$@" 1 "text"
-}
-
-assertFileContains()
-{
-  _assertContains "$@" 0 "file"
-}
-
-assertFileNotContains()
-{
-  _assertContains "$@" 1 "file"
-}
-
-command_exists () {
-    type "$1" > /dev/null 2>&1 ;
-}
-
-assertFileMD5()
-{
-  expectedHash=$1
-  filename=$2
-
-  if command_exists "md5sum"; then
-    md5_cmd="md5sum ${filename}"
-    expected_md5_cmd_output="${expectedHash}  ${filename}"
-  elif command_exists "md5"; then
-    md5_cmd="md5 ${filename}"
-    expected_md5_cmd_output="MD5 (${filename}) = ${expectedHash}"
-  else
-    fail "no suitable MD5 hashing command found on this system"
-  fi
-
-  assertEquals "${expected_md5_cmd_output}" "`${md5_cmd}`"
-}