Release v174 (#1023)

They are now displayed in the build output (instead of being sent to
`/dev/null`) and fail the build early instead of failing later in
`bin/steps/pip-install`.

Fixes #1002.

.github/CODEOWNERS

@@ -1 +1 @@
-* @heroku/languages
+* @heroku/languages

.github/workflows/check_changelog.yml

@@ -1,12 +1,17 @@
 name: Check Changelog
 
-on: [pull_request]
+on:
+  pull_request:
+    types: [opened, reopened, edited, synchronize]
 
 jobs:
- build:
+  check:
-   runs-on: ubuntu-latest
+    runs-on: ubuntu-latest
-   steps:
+    if: |
-   - uses: actions/checkout@v1
+      !contains(github.event.pull_request.body, '[skip changelog]') &&
-   - name: Check that CHANGELOG is touched
+      !contains(github.event.pull_request.body, '[changelog skip]') &&
-     run: |
+      !contains(github.event.pull_request.body, '[skip ci]')
-       cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
+    steps:
+      - uses: actions/checkout@v1
+      - name: Check that CHANGELOG is touched
+        run: git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md

.gitignore

@@ -10,3 +10,5 @@ buildpack/*
 
 builds/dockerenv.staging*
 builds/dockerenv.production
+
+test/scratch

.travis.yml

@@ -1,39 +1,44 @@
-language: ruby
+language: minimal
-dist: trusty
+dist: bionic
-sudo: required
+branches:
-
+  only:
-rvm:
+  - master
-- 2.4.4
-before_script:
-  - gem install bundler -v 1.16.2
-  - bundle exec hatchet ci:setup
 
 script:
-  - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
+  - make test STACK="${STACK}" TEST_CMD="${TEST_CMD}"
-  - docker run --rm -e "STACK=cedar-14" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-cedar-14 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-  - docker run --rm -e "STACK=heroku-16" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-16 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-  - docker run --rm -e "STACK=heroku-18" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-18 bash $TESTFOLDER
 
 jobs:
   include:
-  - stage: Bash linting (shellcheck)
+  - name: Bash linting (shellcheck)
-    sudo: false
     script: make check
-  - stage: Hatchet Integration
+  - name: Hatchet integration tests
-    if: branch = master
+    if: env(TRAVIS_PULL_REQUEST_SLUG) = env(TRAVIS_REPO_SLUG)
-    name: Run Hatchet
+    language: ruby
-    script: "bundle exec rspec"
+    rvm:
+      - 2.6.6
+    before_script:
+      - gem install bundler -v 1.16.2
+    script:
+      - bundle exec hatchet ci:setup
+      - PARALLEL_SPLIT_TEST_PROCESSES=11 bundle exec parallel_split_test spec/hatchet/
+
 env:
-  matrix:
+  jobs:
-    - TESTFOLDER=test/run-deps
+    - STACK=cedar-14 TEST_CMD=test/run-deps
-    - TESTFOLDER=test/run-versions
+    - STACK=cedar-14 TEST_CMD=test/run-versions
-    - TESTFOLDER=test/run-features
+    - STACK=cedar-14 TEST_CMD=test/run-features
+
+    - STACK=heroku-16 TEST_CMD=test/run-deps
+    - STACK=heroku-16 TEST_CMD=test/run-versions
+    - STACK=heroku-16 TEST_CMD=test/run-features
+
+    - STACK=heroku-18 TEST_CMD=test/run-deps
+    - STACK=heroku-18 TEST_CMD=test/run-versions
+    - STACK=heroku-18 TEST_CMD=test/run-features
   global:
     - HATCHET_RETRIES=3
     - IS_RUNNING_ON_CI=true
-    - HATCHET_APP_LIMIT=5
+    - HATCHET_APP_LIMIT=80
     - HATCHET_DEPLOY_STRATEGY=git
     - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
     - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=

CHANGELOG.md

@@ -2,10 +2,84 @@
 
 # Master
 
-- Bug fix: fragile sqlite3 install
+
+# 174 (2020-07-30)
+
+- For repeat builds, also manage the installed versions of setuptools/wheel, rather than just that of pip (#1007).
+- Install an explicit version of wheel rather than the latest release at the time (#1007).
+- Output the installed version of pip, setuptools and wheel in the build log (#1007).
+- Errors installing pip/setuptools/wheel are now displayed in the build output and fail the build early (#1007).
+- Install pip using itself rather than `get-pip.py` (#1007).
+- Disable pip's version check + cache when installing pip/setuptools/wheel (#1007).
+- Install setuptools from PyPI rather than a vendored copy (#1007).
+- Reduce the number of environment variables exposed to `bin/{pre,post}_compile` and other subprocesses (#1011)
+
+# 173 (2020-07-21)
+
+- Python 3.8.5 is now available (CPython)
+
+# 172 (2020-07-17)
+
+- Python 3.8.4 is now available (CPython)
+
+# 171 (2020-07-07)
+
+- Python 3.6.11 and 3.7.8 are now available (CPython).
+
+# 170 (2020-05-19)
+
+- Python 2.7.18, 3.5.9, 3.7.7 and 3.8.3 are now available (CPython).
+- PyPy 2.7 and 3.6, version 7.3.1 are now available (Note: PyPy support is in beta)
+- Docs: Fix explanation of runtime.txt generation when using pipenv
+- Bugfix: Correctly detect Python version when using a `python_version` of `3.8` in `Pipfile.lock`
 
 --------------------------------------------------------------------------------
 
+# 169 (2020-04-22)
+
+- Add a Hatchet test for python 3.8.2
+- Set Code Owners to @heroku/langauges
+- Bugfix: Caching on subsequent redeploys
+- Update tests to support latest version of Python
+
+# 168 (2020-04-06)
+
+- Doc: Update Readme with version numbers
+- update Code Owners to include the Heroku Buildpack Maintainers team
+- Deprecation warning: BUILD_WITH_GEO_LIBRARIES is now deprecated. See warning for details.
+- Clean up build log output
+- Update Python versions in README to match docs
+- Django version detection fixed, link updated
+
+# 167 (2020-03-26)
+
+- Add failcase for cache busting
+- Bugfix: Clearing pip dependencies
+
+# 166 (2020-03-05)
+
+- Correct ftp to https in vendored file
+- Warn for Django 1.11 approaching EOL, provide link to roadmap
+
+# 165 (2020-02-27)
+
+- Python 3.8.2 now available.
+
+# 164 (2020-02-20)
+
+- Update requirements.txt builds to use Pip 20.0.2
+- Download get-pip.py to tmpdir instead of root dir
+
+# 163 (2019-12-23)
+
+- New pythons released:
+  Python 3.8.1, 3.7.6, 3.6.10 (CPython)
+  Beta Release: Pypy 2.7 and 3.6, version 7.2.0
+
+# 162 (2019-12-06)
+
+- Bug fix: fragile sqlite3 install
+
 # 161 (2019-12-2)
 
 - Bug fix: Sqlite3 version bump

Gemfile

@@ -4,3 +4,5 @@ gem "rspec"
 gem "heroku_hatchet"
 gem "rspec-retry"
 gem "rake"
+gem "parallel_split_test"
+

Gemfile.lock

@@ -1,21 +1,22 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.1)
+    activesupport (6.0.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    concurrent-ruby (1.1.3)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    concurrent-ruby (1.1.6)
     diff-lcs (1.3)
     erubis (2.7.0)
-    excon (0.62.0)
+    excon (0.73.0)
     heroics (0.0.25)
       erubis (~> 2.0)
       excon
       moneta
       multi_json (>= 1.9.2)
-    heroku_hatchet (4.0.6)
+    heroku_hatchet (5.0.3)
       excon (~> 0)
       minitest-retry (~> 0.1.9)
       platform-api (~> 2)
@@ -23,17 +24,21 @@ GEM
       rrrretry (~> 1)
       thor (~> 0)
       threaded (~> 0)
-    i18n (1.1.1)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    minitest (5.11.3)
+    minitest (5.14.1)
     minitest-retry (0.1.9)
       minitest (>= 5.0)
     moneta (1.0.0)
-    multi_json (1.13.1)
+    multi_json (1.14.1)
+    parallel (1.19.1)
+    parallel_split_test (0.7.0)
+      parallel (>= 0.5.13)
+      rspec (>= 3.1.0)
     platform-api (2.2.0)
       heroics (~> 0.0.25)
       moneta (~> 1.0.0)
-    rake (12.3.1)
+    rake (12.3.3)
     repl_runner (0.0.3)
       activesupport
     rrrretry (1.0.0)
@@ -55,17 +60,19 @@ GEM
     thor (0.20.3)
     thread_safe (0.3.6)
     threaded (0.0.4)
-    tzinfo (1.2.5)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
+    zeitwerk (2.3.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   heroku_hatchet
+  parallel_split_test
   rake
   rspec
   rspec-retry
 
 BUNDLED WITH
-   1.16.3
+   2.1.4

Makefile

@@ -1,26 +1,26 @@
 # These targets are not files
-.PHONY: tests
+.PHONY: check test buildenv-heroku-16 buildenv-heroku-18 tools
 
-test: test-heroku-18 test-heroku-16 test-cedar-14
+STACK ?= heroku-18
+TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
+
+ifeq ($(STACK),cedar-14)
+	# Cedar-14 doesn't have a build image varient.
+	IMAGE_TAG := heroku/cedar:14
+else
+	# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
+	IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
+endif
 
 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
 	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
 	@shellcheck -x bin/steps/hooks/*
 
-test-cedar-14:
+test:
-	@echo "Running tests in docker (cedar-14)..."
+	@echo "Running tests using: STACK=$(STACK) TEST_CMD='$(TEST_CMD)'"
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
-
+	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
-test-heroku-16:
-	@echo "Running tests in docker (heroku-16)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
-	@echo ""
-
-test-heroku-18:
-	@echo "Running tests in docker (heroku-18)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
 
 buildenv-heroku-16:

NOTICE

@@ -86,31 +86,6 @@ Each version is given a distinguishing version number. If the Library as you rec
 
 If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library.
 
-get-pip.py license
-------------------
-
-Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-
 pip-pop license
 ---------------
 
@@ -134,4 +109,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+THE SOFTWARE.

Pipfile

@@ -1,9 +0,0 @@
-[[source]]
-url = "https://pypi.python.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[dev-packages]
-
-[packages]
-"bob-builder" = "==0.0.13"

Pipfile.lock

@@ -1,52 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "36d17c46a8b1b844b3cae475f6f42d6c0a9e59b2a9685cbcdc0985656a7a129f"
-        },
-        "host-environment-markers": {
-            "implementation_name": "cpython",
-            "implementation_version": "3.6.3",
-            "os_name": "posix",
-            "platform_machine": "x86_64",
-            "platform_python_implementation": "CPython",
-            "platform_release": "16.7.0",
-            "platform_system": "Darwin",
-            "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64",
-            "python_full_version": "3.6.3",
-            "python_version": "3.6",
-            "sys_platform": "darwin"
-        },
-        "pipfile-spec": 6,
-        "requires": {},
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.python.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "bob-builder": {
-            "hashes": [
-                "sha256:b4de49a8e436fcaf82236ea43f78413b4a4c92100726e382ab57b6bdfb38fe64",
-                "sha256:288e3e765c4890fe9a63ae52ac6b4a963c13fe508482c70ff701a5ae21b9a673"
-            ],
-            "version": "==0.0.13"
-        },
-        "boto": {
-            "hashes": [
-                "sha256:13be844158d1bd80a94c972c806ec8381b9ea72035aa06123c5db6bc6a6f3ead",
-                "sha256:deb8925b734b109679e3de65856018996338758f4b916ff4fe7bb62b6d7000d1"
-            ],
-            "version": "==2.48.0"
-        },
-        "docopt": {
-            "hashes": [
-                "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
-            ],
-            "version": "==0.6.2"
-        }
-    },
-    "develop": {}
-}

README.md

@@ -46,7 +46,8 @@ To specify your python version, you also need a `runtime.txt` file - unless you
 
 Current default Python Runtime: Python 3.6.9
 
-Alternatively, you can provide a `setup.py` file, or a `Pipfile`. Using `Pipenv` will generate `runtime.txt` based on `python-version` at build time.
+Alternatively, you can provide a `setup.py` file, or a `Pipfile`.
+Using `pipenv` will generate `runtime.txt` at build time if one of the field `python_version` or `python_full_version` is specified in the `requires` section of your `Pipfile`.
 
 Specify a Buildpack Version
 ---------------------------
@@ -61,26 +62,32 @@ Specify a Python Runtime
 
 Supported runtime options include:
 
-- `python-3.7.4`
+- `python-3.8.5`
-- `python-3.6.9`
+- `python-3.7.8`
-- `python-2.7.17`
+- `python-3.6.11`
+- `python-2.7.18`
 
 ## Tests
 
 The buildpack tests use [Docker](https://www.docker.com/) to simulate
 Heroku's [stack images.](https://devcenter.heroku.com/articles/stack)
 
-To run the test suite:
+To run the test suite against the default stack:
 
 ```
 make test
 ```
 
-Or to test in a particular stack:
+Or to test against a particular stack:
 
 ```
-make test-heroku-18
+make test STACK=heroku-16
-make test-heroku-16
+```
+
+To run only a subset of the tests:
+
+```
+make test TEST_CMD=tests/versions
 ```
 
 The tests are run via the vendored

bin/compile

@@ -16,7 +16,9 @@
 set -eo pipefail
 
 # Boostrap the Buildpack Standard Library.
-export BPLOG_PREFIX="buildpack.python"
+# Disable unused env var warning since shellcheck doesn't know about the stdlib.
+# shellcheck disable=2034
+BPLOG_PREFIX="buildpack.python"
 export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
 
 [ "$BUILDPACK_XTRACE" ] && set -o xtrace
@@ -58,22 +60,18 @@ PY36="python-3.6"
 PY35="python-3.5"
 PY34="python-3.4"
 PY27="python-2.7"
+PYPY27="pypy2.7"
+PYPY36="pypy3.6"
 
 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
-# If pip doesn't match this version (the version we install), run the installer.
-PIP_UPDATE="9.0.2"
-
-export DEFAULT_PYTHON_STACK PIP_UPDATE
-export PY37 PY36 PY35 PY27 PY34
 
 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.
 # The `warnings` subscript then greps through this for common problems and guides
 # the user towards resolution of known issues.
 WARNINGS_LOG=$(mktemp)
-export WARNINGS_LOG
+RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
-export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
 
 # The buildpack ships with a few executable tools (e.g. pip-grep, etc).
 # This installs them into the path, so we can execute them directly.
@@ -155,7 +153,7 @@ cd "$BUILD_DIR"
 # - Once the build is complete, `~/.heroku/{known-paths}` is copied back into the cache.
 
 # Create the cache directory, if it doesn't exist.
-mkdir -p "$CACHE_DIR"
+mkdir -p "$CACHE_DIR/.heroku"
 
 # Restore old artifacts from the cache.
 mkdir -p .heroku
@@ -197,8 +195,6 @@ else
   CACHED_PYTHON_STACK=$STACK
 fi
 
-export CACHED_PYTHON_STACK
-
 # Pipenv Python version support.
 # Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version).
 # Convert it to a runtime.txt file.
@@ -239,17 +235,8 @@ mtime "python.install.time" "${start}"
 # shellcheck source=bin/steps/pipenv
 source "$BIN_DIR/steps/pipenv"
 
-# Uninstall removed dependencies with Pip.
-# The buildpack will automatically remove any declared dependencies (in requirements.txt)
-# that were explicitly removed. This machinery is a bit complex, but it is not complicated.
-(( start=$(nowms) ))
-# shellcheck source=bin/steps/pip-uninstall
-source "$BIN_DIR/steps/pip-uninstall"
-mtime "pip.uninstall.time" "${start}"
-
 # If no requirements.txt file given, assume `setup.py develop` is intended.
 # This allows for people to ship a setup.py application to Heroku
-# (which is rare, but I vouch that it should work!)
 
 if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
   echo "-e ." > requirements.txt

bin/default_pythons

@@ -1,11 +1,16 @@
 #!/usr/bin/env bash
 
-DEFAULT_PYTHON_VERSION="python-3.6.9"
+# Disable unused env var warning, since shellcheck doesn't take into account
-LATEST_38="python-3.8.0"
+# that this file is sourced. We don't want to use export since it exposes
-LATEST_37="python-3.7.5"
+# the env vars to subprocesses.
-LATEST_36="python-3.6.9"
+# shellcheck disable=2034
-LATEST_35="python-3.5.7"
-LATEST_34="python-3.4.10"
-LATEST_27="python-2.7.17"
 
-export DEFAULT_PYTHON_VERSION LATEST_38 LATEST_37 LATEST_36 LATEST_35 LATEST_34 LATEST_27
+DEFAULT_PYTHON_VERSION="python-3.6.11"
+LATEST_38="python-3.8.5"
+LATEST_37="python-3.7.8"
+LATEST_36="python-3.6.11"
+LATEST_35="python-3.5.9"
+LATEST_34="python-3.4.10"
+LATEST_27="python-2.7.18"
+PYPY_36="pypy3.6-7.3.1"
+PYPY_27="pypy2.7-7.3.1"

bin/steps/README.MD

@@ -0,0 +1,21 @@
+# Python Buildpack Install Steps
+
+TODO: Add context on Python install steps, such as why symlinking vs copying
+
+## Installing Python packages using Pip
+
+### Convention: Use `python` process to invoke Pip
+
+We don't use this convention (yet) but this is an upcoming change being considered.
+
+This is a bigger concern on Windows than it is in Linux environments, but an
+emerging convention in the Python community is to invoke pip using:
+
+```
+python3 -m pip [options]
+```
+
+Invoking pip this way ensures correct location - python knows where these
+packages are stored because it put them there (defaults to Python's pathing info).
+
+All normal command line options are available using this method.

bin/steps/gdal

@@ -22,6 +22,10 @@ source "$BIN_DIR/utils"
 if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
 
   if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then
+
+    puts-warn "The vendored GDAL package in the Heroku Python Buildpack now deprecated."
+    puts-warn "To enable GDAL use an alternative buildpack is available here - https://github.com/heroku/heroku-geo-buildpack"
+
     echo "-----> Noticed GDAL. Bootstrapping gdal."
     mkdir -p .heroku/vendor
     # Download and extract cryptography into target vendor directory.

bin/steps/geo-libs

@@ -24,6 +24,9 @@ source "$BIN_DIR/utils"
 if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
   mcount "buildvar.BUILD_WITH_GEO_LIBRARIES"
 
+  puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."
+  puts-warn "An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
+
   if [ ! -f ".heroku/vendor/bin/proj" ]; then
     echo "-----> Bootstrapping gdal, geos, proj."
     mkdir -p .heroku/vendor

bin/steps/pip-install

@@ -38,9 +38,16 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
         mcount "failure.none-version"
     fi
 
+    if grep -qi '^django==1.*' requirements.txt; then
+        puts-warn "Your Django version is nearing the end of its community support."
+        puts-warn "Upgrade to continue to receive security updates and for the best experience with Django."
+        puts-warn "For more information, check out https://www.djangoproject.com/download/#supported-versions"
+    fi
+
     if [ ! -f "$BUILD_DIR/.heroku/python/bin/pip" ]; then
         exit 1
     fi
+
     /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
     PIP_STATUS="${PIPESTATUS[0]}"
     set -e
@@ -52,12 +59,9 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
         exit 1
     fi
 
-    # Smart Requirements handling
     cp requirements.txt .heroku/python/requirements-declared.txt
     /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
 
-    echo
-
     # Install test dependencies, for CI.
     if [ "$INSTALL_TEST" ]; then
         if [[ -f "$1/requirements-test.txt" ]]; then

bin/steps/pipenv-python-version

@@ -27,6 +27,9 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
                 if [ "$PYTHON" = 3.7 ]; then
                     echo "$LATEST_37" > "$BUILD_DIR/runtime.txt"
                 fi
+                if [ "$PYTHON" = 3.8 ]; then
+                    echo "$LATEST_38" > "$BUILD_DIR/runtime.txt"
+                fi
             fi
 
 

bin/steps/python

@@ -9,6 +9,10 @@ VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"
 
 SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
 
+ONLY_SUPPORTED_2_VERSION="Only the latest version of Python 2 is supported on the platform. Please consider upgrading to"
+
+PYTHON_2_EOL_UPDATE="Python 2 has reached it's community EOL. Upgrade your Python runtime to maintain a secure application as soon as possible."
+
 # check if runtime exists
 if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
   if [[ "$PYTHON_VERSION" == $PY38* ]]; then
@@ -48,8 +52,26 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
   fi
   if [[ "$PYTHON_VERSION" == $PY27* ]]; then
     # security update note
+    if [[ "$(date "+%Y")" -gt "2019" ]]; then
+      puts-warn "$PYTHON_2_EOL_UPDATE"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq"
+    fi
     if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then
-      puts-warn "$SECURITY_UPDATE" "$LATEST_27"
+      puts-warn "$ONLY_SUPPORTED_2_VERSION" "$LATEST_27"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PYPY27* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$PYPY_27" ]; then
+      puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_27}?"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PYPY36* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$PYPY_36" ]; then
+      puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_36}?"
       echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
     fi
   fi
@@ -64,14 +86,6 @@ if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
     rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
 fi
 
-# need to clear the cache for first time installing SQLite3,
-# since the version is changing and could lead to runtime errors
-# with compiled extensions.
-if [ -d .heroku/python ] && [ ! -f .heroku/python-sqlite3-version ] && python_sqlite3_check "$PYTHON_VERSION"; then
-  puts-step "Need to update SQLite3, clearing cache"
-  rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
-fi
-
 if [ -f .heroku/python-version ]; then
   if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then
       puts-step "Found $(cat .heroku/python-version), removing"
@@ -81,6 +95,24 @@ if [ -f .heroku/python-version ]; then
   fi
 fi
 
+# Check if we should reinstall python dependencies
+if [[ ! -f "$CACHE_DIR/.heroku/requirements.txt" ]]; then
+  # IF there's no cached dependencies, update cached version of requirements.txt
+  # This should only run for new apps and first deploys after this update
+  cp -R "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt"
+else
+  # IF there IS a cached directory, check for differences with the new one
+  if ! diff "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt" &> /dev/null; then
+    puts-step "Requirements file has been changed, clearing cached dependencies"
+    # if there are any differences, clear the Python cache
+    # Installing Python over again does not take noticably more time
+    cp -R "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt"
+    rm -rf .heroku/python
+    unset SKIP_INSTALL
+  else
+    puts-step "No change in requirements detected, installing from cache"
+  fi
+fi
 
 if [ ! "$SKIP_INSTALL" ]; then
     puts-step "Installing $PYTHON_VERSION"
@@ -99,33 +131,52 @@ if [ ! "$SKIP_INSTALL" ]; then
   # Record for future reference.
   echo "$PYTHON_VERSION" > .heroku/python-version
   echo "$STACK" > .heroku/python-stack
-  FRESH_PYTHON=true
 
   hash -r
 fi
 
-# Heroku uses the get-pip utility maintained by the Python community to vendor Pip.
-# https://github.com/pypa/get-pip
-GETPIP="https://lang-python.s3.amazonaws.com/etc/get-pip.py"
-
-if ! curl -s "${GETPIP}" -o "$ROOT_DIR/get-pip.py" &> /dev/null; then
-    mcount "failure.python.get-pip"
-    echo "Failed to pull down get-pip"
-    exit 1
-fi
-
-# If Pip isn't up to date:
-if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
-
-  puts-step "Installing pip"
-
-  # Remove old installations.
-  rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
-  rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
-
-  /app/.heroku/python/bin/python "$ROOT_DIR/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null
-  /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
-fi
-
 set -e
+
+PIP_VERSION='20.0.2'
+SETUPTOOLS_VERSION='39.0.1'
+WHEEL_VERSION='0.34.2'
+
+if [[ "${PYTHON_VERSION}" == ${PY34}* ]]; then
+  # Python 3.4 support was dropped in pip 19.2+ and wheel 0.34.0+.
+  PIP_VERSION='19.1.1'
+  WHEEL_VERSION='0.33.6'
+fi
+
+# We don't use get-pip.py, since:
+#  - it uses `--force-reinstall`, which is unnecessary here and slows down repeat builds
+#  - it means downloading pip twice (once embedded in get-pip.py, and again during
+#    the install, since get-pip.py can't install the embedded version directly)
+#  - we would still have to manage several versions of get-pip.py, to support older Pythons.
+# Instead, we use the pip wheel to install itself, using the method described here:
+# https://github.com/pypa/pip/issues/2351#issuecomment-69994524
+PIP_WHEEL_FILENAME="pip-${PIP_VERSION}-py2.py3-none-any.whl"
+PIP_WHEEL_URL="https://lang-python.s3.amazonaws.com/common/${PIP_WHEEL_FILENAME}"
+PIP_WHEEL="${TMPDIR:-/tmp}/${PIP_WHEEL_FILENAME}"
+
+if ! curl -sSf "${PIP_WHEEL_URL}" -o "$PIP_WHEEL"; then
+  mcount "failure.python.download-pip"
+  puts-warn "Failed to download pip"
+  exit 1
+fi
+
+if [[ -f "$BUILD_DIR/Pipfile" ]]; then
+  # The buildpack is pinned to old pipenv, which requires older pip.
+  # Pip 9.0.2 doesn't support installing itself from a wheel, so we have to use split
+  # versions here (ie: installer pip version different from target pip version).
+  PIP_VERSION='9.0.2'
+  PIP_TO_INSTALL="pip==${PIP_VERSION}"
+else
+  PIP_TO_INSTALL="${PIP_WHEEL}"
+fi
+
+puts-step "Installing pip ${PIP_VERSION}, setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}"
+
+/app/.heroku/python/bin/python "${PIP_WHEEL}/pip" install --quiet --disable-pip-version-check --no-cache \
+  "${PIP_TO_INSTALL}" "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}"
+
 hash -r

bin/steps/sqlite3

@@ -67,7 +67,6 @@ buildpack_sqlite3_install() {
   puts-step "Installing SQLite3"
 
   if sqlite3_install "$BUILD_DIR/.heroku/python" ; then
-    echo "Sqlite3 successfully installed."
     mcount "success.python.sqlite3"
   else
     echo "Sqlite3 failed to install."

builds/heroku-18.Dockerfile

@@ -1,4 +1,4 @@
-FROM heroku/heroku:18-build.v16
+FROM heroku/heroku:18-build
 
 WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \

builds/libraries/vendor/libffi

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Skip the build for heroku-16.
-if [[ $S3_PREFIX == "heroku-16" ]]; then
-    exit 0
-fi
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building libffi…"
-
-SOURCE_TARBALL='ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz'
-
-curl -L $SOURCE_TARBALL | tar x
-
-cd libffi-3.1
-./configure --prefix=$OUT_PREFIX --disable-static &&
-make
-make install
-
-# Cleanup
-cd ..

builds/libraries/vendor/libmemcache

@@ -3,9 +3,9 @@
 
 OUT_PREFIX=$1
 
-# Skip the build for heroku-16.
+if [[ $S3_PREFIX != "cedar-14" ]]; then
-if [[ $S3_PREFIX == "heroku-16" ]]; then
+    echo "libmemcached only needs to be built for cedar-14, since newer stacks include it in the base image"
-    exit 0
+    exit 1
 fi
 
 # fail hard

builds/runtimes/pypy2.7

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# fail hard
+set -o pipefail
+# fail harder
+set -eu
+
+OUT_PREFIX=$1
+
+dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
+dep_name=$(basename $BASH_SOURCE) # this is us
+dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name
+dep_package=${dep_name}-v${dep_version} # it's always "pypy2-…"
+dep_dirname=${dep_package}-linux64
+dep_archive_name=${dep_dirname}.tar.bz2
+dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+
+echo "Building PyPy…"
+echo "${dep_url}"
+
+curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+
+ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python"

builds/runtimes/pypy2.7-7.2.0

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy2.7

builds/runtimes/pypy2.7-7.3.1

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy2.7

builds/runtimes/pypy3.6

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# fail hard
+set -o pipefail
+# fail harder
+set -eu
+
+OUT_PREFIX=$1
+
+dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
+dep_name=$(basename $BASH_SOURCE) # this is us
+dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name
+dep_package=${dep_name}${dep_version_prefix:-}-v${dep_version}${dep_version_suffix:-}
+dep_dirname=${dep_package}-linux64
+dep_archive_name=${dep_dirname}.tar.bz2
+dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+
+echo "Building PyPy3…"
+echo "${dep_url}"
+
+curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+
+ln "$OUT_PREFIX/bin/pypy3" "$OUT_PREFIX/bin/python"

builds/runtimes/pypy3.6-7.2.0

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy3.6

builds/runtimes/pypy3.6-7.3.1

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy3.6

builds/runtimes/python-2.7.18

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/2.7.18/Python-2.7.18.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-2.7.18 src
+cd src
+
+./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +

builds/runtimes/python-3.5.9

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.6.10

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.6.11

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.7.6

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.7.7

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.7.8

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.1

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.2

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.3

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.4

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.5

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python3

@@ -16,6 +16,9 @@ python_version=${BASE^} # this gives us only the filename with version number
 version_number=$(echo "$python_version" | cut -d- -f2) # this returns just X.X.X
 dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz
 
+echo "Building Python 3..."
+echo "Pulling from source: ${dep_url}"
+
 curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}"
 mv "${OUT_PREFIX}/${python_version}" src
 cd src
@@ -31,9 +34,4 @@ find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -
 # Remove spare /
 LOCATION=${OUT_PREFIX%?}
 
-# Create links to SQLITE headers so Python can call them at runtime
-mkdir -p ${OUT_PREFIX}/include ${OUT_PREFIX}/lib/x86_64-linux-gnu
-cp /usr/include/sqlite3*.h ${OUT_PREFIX}/include
-ln -fs $(realpath /usr/lib/x86_64-linux-gnu/libsqlite3.so) ${OUT_PREFIX}/lib/x86_64-linux-gnu/libsqlite3.so
-
 ln $LOCATION/bin/python3 $LOCATION/bin/python

hatchet.json

@@ -1,5 +1,6 @@
 {
   "python": [
-    "heroku/python-getting-started"
+    "heroku/python-getting-started",
+    "sharpstone/python_default"
   ]
 }

hatchet.lock

@@ -1,3 +1,5 @@
 ---
 - - "./repos/python/python-getting-started"
-  - f56b90499ec11e1c9576da2f8c7331300e189db3
+  - master
+- - "./repos/python/python_default"
+  - ca947f69027b2a30be5d26f9a42f25e54f4d7a1a

requirements.txt

@@ -1,3 +1,3 @@
 docopt==0.6.2
-bob-builder
+bob-builder==0.0.17
 boto==2.48.0

spec/hatchet/ci_spec.rb

@@ -0,0 +1,33 @@
+require_relative '../spec_helper'
+
+describe "Heroku CI" do
+  it "works" do
+    before_deploy = Proc.new do
+      File.open("app.json", "w+") do |f|
+        f.puts <<~EOM
+          {
+            "environments": {
+              "test": {
+                "scripts": {
+                  "test": "nosetests"
+                }
+              }
+            }
+          }
+        EOM
+      end
+
+      run!("echo nose >> requirements.txt")
+    end
+
+    Hatchet::Runner.new("python_default", before_deploy: before_deploy).run_ci do |test_run|
+      expect(test_run.output).to match("Downloading nose")
+      expect(test_run.output).to match("OK")
+
+      test_run.run_again
+
+      expect(test_run.output).to match("installing from cache")
+      expect(test_run.output).to_not match("Downloading nose")
+    end
+  end
+end

spec/hatchet/python_spec.rb

@@ -1,10 +1,72 @@
 require_relative '../spec_helper'
 
-describe "Python!!!!!!!!!!!" do
+describe "Python" do
-  it "🐍" do
+  describe "cache" do
-    Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app|
+    it "functions correctly" do
-      expect(app.output).to           match(/Installing pip/)
+      Hatchet::Runner.new("python_default").deploy do |app|
-      expect(app.run('python -V')).to match(/3.6.9/)
+        expect(app.output).to match(/Installing pip/)
+
+        expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies")
+        expect(app.output).to_not match("No change in requirements detected, installing from cache")
+        expect(app.output).to_not match("No such file or directory")
+        expect(app.output).to_not match("cp: cannot create regular file")
+
+        # Redeploy with changed requirements file
+        run!(%Q{echo "" >> requirements.txt})
+        run!(%Q{echo "pygments" >> requirements.txt})
+        run!(%Q{git add . ; git commit --allow-empty -m next})
+        app.push!
+
+        # Check the cache to have cleared
+        expect(app.output).to match("Requirements file has been changed, clearing cached dependencies")
+        expect(app.output).to_not match("No dependencies found, preparing to install")
+        expect(app.output).to_not match("No change in requirements detected, installing from cache")
+
+        # With no changes on redeploy, the cache should be present
+        run!(%Q{git commit --allow-empty -m next})
+        app.push!
+
+        expect(app.output).to match("No change in requirements detected, installing from cache")
+        expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies")
+        expect(app.output).to_not match("No dependencies found, preparing to install")
+      end
+    end
+  end
+
+  describe "python versions" do
+    let(:stack) { ENV["HEROKU_TEST_STACK"] || DEFAULT_STACK }
+    it "works with 3.7.6" do
+      version = "3.7.6"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack).deploy do |app|
+        expect(app.run('python -V')).to match(version)
+      end
+    end
+
+    it "works with 3.8.2" do
+      version = "3.8.2"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack).deploy do |app|
+        expect(app.run('python -V')).to match(version)
+      end
+    end
+
+    it "fails with a bad version" do
+      version = "3.8.2.lol"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack, allow_failure: true).deploy do |app|
+        expect(app.output).to match("not available for this stack")
+      end
+    end
+  end
+
+  it "getting started app has no relative paths" do
+    buildpacks = [
+      :default,
+      "https://github.com/sharpstone/force_absolute_paths_buildpack"
+    ]
+    Hatchet::Runner.new("python-getting-started", buildpacks: buildpacks).deploy do |app|
+      # Deploy works
     end
   end
 end

spec/spec_helper.rb

@@ -1,4 +1,4 @@
-ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/' + ENV['TRAVIS_REPO_SLUG'] + '.git'
+ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/heroku/heroku-buildpack-python.git'
 
 require 'rspec/core'
 require 'rspec/retry'
@@ -15,9 +15,10 @@ RSpec.configure do |config|
   end
 end
 
-if ENV['TRAVIS']
+DEFAULT_STACK = 'heroku-18'
-  # Don't execute tests against "merge" commits
-  exit 0 if ENV['TRAVIS_PULL_REQUEST'] != 'false' && ENV['TRAVIS_BRANCH'] == 'master'
-end
 
-DEFAULT_STACK = 'heroku-16'
+def run!(cmd)
+  out = `#{cmd}`
+  raise "Error running command #{cmd} with output: #{out}" unless $?.success?
+  return out
+end

test/fixtures/hooks/bin/post_compile

@@ -0,0 +1,4 @@
+set -euo pipefail
+
+echo "post_compile ran!"
+echo "post_compile env: $(printenv | cut -d '=' -f 1 | sort | xargs)."

test/fixtures/hooks/bin/pre_compile

@@ -0,0 +1,4 @@
+set -euo pipefail
+
+echo "pre_compile ran!"
+echo "pre_compile env: $(printenv | cut -d '=' -f 1 | sort | xargs)."

test/fixtures/not-old-django/requirements.txt

@@ -0,0 +1,2 @@
+pwned-passwords-django==1.4
+Django==2.1

test/fixtures/old-django/requirements.txt

@@ -0,0 +1 @@
+Django==1.11

test/fixtures/pypy2_7/requirements.txt

@@ -0,0 +1 @@
+django

test/fixtures/pypy2_7/runtime.txt

@@ -0,0 +1 @@
+pypy2.7-7.2.0

test/fixtures/pypy3_6/requirements.txt

@@ -0,0 +1 @@
+django

test/fixtures/pypy3_6/runtime.txt

@@ -0,0 +1 @@
+pypy3.6-7.2.0

test/fixtures/python2/runtime.txt

@@ -1 +1 @@
-python-2.7.17
+python-2.7.18

test/fixtures/python3_5/runtime.txt

@@ -1 +1 @@
-python-3.5.7
+python-3.5.9

test/fixtures/python3_8/runtime.txt

@@ -1 +1 @@
-python-3.8.0
+python-3.8.2

test/fixtures/python3_8_warn/requirements.txt

@@ -0,0 +1 @@
+requests

test/fixtures/python3_8_warn/runtime.txt

@@ -0,0 +1 @@
+python-3.8.0

test/run-deps

@@ -5,8 +5,9 @@
 source "bin/default_pythons"
 
 testAirflow() {
-  export SLUGIFY_USES_TEXT_UNIDECODE="yes"
+  local env_dir="$(mktmpdir)"
-  compile "airflow"
+  echo 'yes' > "${env_dir}/SLUGIFY_USES_TEXT_UNIDECODE"
+  compile 'airflow' '' "${env_dir}"
   assertCaptured "apache-airflow==1.10.2"
   assertCapturedSuccess
 }
@@ -17,19 +18,30 @@ testCollectstatic() {
 }
 
 testGEOS() {
-  export BUILD_WITH_GEO_LIBRARIES=1
+  local env_dir="$(mktmpdir)"
-  compile "geos"
+  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
+  compile 'geos' '' "${env_dir}"
   assertCaptured "geos"
   assertCapturedSuccess
 }
 
+testGEOSDeprecation() {
+  local env_dir="$(mktmpdir)"
+  echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
+  compile 'geos' '' "${env_dir}"
+  assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated.
+ !     An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
+  assertCapturedSuccess
+}
+
 testNLTK() {
+  local env_dir="$(mktmpdir)"
   # NOTE: This is a RuntimeWarning emitted by Python 3's runpy.py script
   # which is what is used when you call `python -m <module>`. This is due to
   # how nltk imports things. It's not actually an error, but it would probably
   # be bad to silence in Production.
-  export PYTHONWARNINGS="ignore::RuntimeWarning"
+  echo 'ignore::RuntimeWarning' > "${env_dir}/PYTHONWARNINGS"
-  compile "nltk"
+  compile 'nltk' '' "${env_dir}"
   assertCaptured "[nltk_data] Downloading package city_database" "STD_ERR"
   assertCapturedSuccess
 }
@@ -48,7 +60,6 @@ testPysqlite() {
 
 testSqliteInstall() {
   compile "pythonDefault"
-  assertCaptured "Sqlite3 successfully installed."
   assertNotCaptured "Sqlite3 failed to install."
   assertCapturedSuccess
 }

test/run-features

@@ -44,6 +44,7 @@ testStandardRequirements() {
 
 testPipenv() {
   compile "pipenv"
+  assertCaptured "Installing pip 9.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCapturedSuccess
 }
 
@@ -74,6 +75,61 @@ testNoRequirements() {
   assertCapturedError
 }
 
+testWarnOldDjango() {
+  compile "old-django"
+  assertCaptured "Your Django version is nearing the end of its community support."
+  assertCapturedSuccess
+}
+
+testDontWarnOldDjango() {
+  compile "not-old-django"
+  assertNotCaptured "Your Django version is nearing the end of its community support."
+  assertCapturedSuccess
+}
+
+testHooks() {
+  # Test that the hooks are called correctly, and that the environment contains
+  # the app's config vars but no unexpected env vars from the buildpack.
+  local env_dir="$(mktmpdir)"
+  echo 'test' > "${env_dir}/SOME_APP_CONFIG_VAR"
+  local expected_env_vars=(
+    _
+    BIN_DIR
+    BUILD_DIR
+    BUILDPACK_LOG_FILE
+    CACHE_DIR
+    C_INCLUDE_PATH
+    CPLUS_INCLUDE_PATH
+    ENV_DIR
+    EXPORT_PATH
+    HOME
+    LANG
+    LD_LIBRARY_PATH
+    LIBRARY_PATH
+    OLDPWD
+    PATH
+    PKG_CONFIG_PATH
+    PROFILE_PATH
+    PWD
+    PYTHONUNBUFFERED
+    SHLVL
+    SOME_APP_CONFIG_VAR
+    STACK
+    VENDOR_URL
+  )
+  if [[ "${STACK}" == "cedar-14" || "${STACK}" == "heroku-16" ]]; then
+    # Remove "OLDPWD" from expected_env_vars since for bash <4.4 it's not exported to subshells:
+    # https://github.com/heroku/heroku-buildpack-python/pull/1011#issuecomment-665117835
+    read -ra expected_env_vars <<< "${expected_env_vars[@]/OLDPWD/}"
+  fi
+  compile 'hooks' '' "${env_dir}"
+  assertCaptured "pre_compile ran!"
+  assertCaptured "pre_compile env: ${expected_env_vars[*]}."
+  assertCaptured "post_compile ran!"
+  assertCaptured "post_compile env: ${expected_env_vars[*]}."
+  assertCapturedSuccess
+}
+
 pushd $(dirname 0) >/dev/null
 popd >/dev/null
 

test/run-versions

@@ -9,6 +9,7 @@ testPythonDefault() {
   compile "pythonDefault"
   assertCaptured $DEFAULT_PYTHON_VERSION
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -18,7 +19,13 @@ testPython2() {
     echo $LATEST_27 > "runtime.txt"
     compile "python2"
     assertCaptured $LATEST_27
+    if [[ $(date "+%Y") > "2019" ]]; then
+      assertCaptured "python-2-7-eol-faq";
+    else
+      assertNotCaptured "python-2-7-eol-faq";
+    fi
     assertNotCaptured "security update"
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
 }
@@ -26,7 +33,12 @@ testPython2() {
 testPython2_warn() {
     compile "python2_warn"
     assertCaptured "python-2.7.15"
-    assertCaptured "security update!"
+    if [[ $(date "+%Y") > "2019" ]]; then
+      assertCaptured "python-2-7-eol-faq";
+    else
+      assertNotCaptured "python-2-7-eol-faq";
+    fi
+    assertCaptured "Only the latest version"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
 }
@@ -41,6 +53,7 @@ testPython3_4() {
   compile "python3_4"
   assertCaptured $LATEST_34
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 19.1.1, setuptools 39.0.1 and wheel 0.33.6"
   # if cedar 14 and legacy binaries, fail. if cedar 14 and staging, succeed.
   if [[ ! -n $USE_STAGING_BINARIES ]] && [[ $STACK == "cedar-14" ]]; then
       assertCapturedError
@@ -76,6 +89,7 @@ testPython3_5() {
   compile "python3_5"
   assertCaptured $LATEST_35
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -98,6 +112,7 @@ testPython3_6() {
   compile "python3_6"
   assertCaptured $LATEST_36
   assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
@@ -124,6 +139,7 @@ testPython3_7() {
   else
     assertNotCaptured "security update"
     assertCaptured $LATEST_37
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
   fi
@@ -147,6 +163,18 @@ testPython3_7_fail() {
   assertCapturedError
 }
 
+testPython3_7_warn() {
+  compile "python3_8_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "python-3.8.0"
+    assertCaptured "security update!"
+    assertCaptured "Installing SQLite3"
+    assertCapturedSuccess
+  fi
+}
+
 testPython3_8() {
   updateVersion "python3_8" $LATEST_38
   compile "python3_8"
@@ -155,6 +183,7 @@ testPython3_8() {
   else
     assertNotCaptured "security update"
     assertCaptured $LATEST_38
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
   fi
@@ -166,6 +195,30 @@ testPython3_8_fail() {
   assertCapturedError
 }
 
+testPypy3_6() {
+  compile "pypy3_6"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "Installing pypy"
+    assertCaptured "$PYPY_36"
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
+    assertCapturedSuccess
+  fi
+}
+
+testPypy2_7() {
+  compile "pypy2_7"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "Installing pypy"
+    assertCaptured "$PYPY_27"
+    assertCaptured "Installing pip 20.0.2, setuptools 39.0.1 and wheel 0.34.2"
+    assertCapturedSuccess
+  fi
+}
+
 pushd $(dirname 0) >/dev/null
 popd >/dev/null
 

test/utils

@@ -56,21 +56,6 @@ resetCapture()
   unset rtrn # deprecated
 }
 
-detect()
-{
-  capture ${BUILDPACK_HOME}/bin/detect ${BUILD_DIR}
-}
-
-compile()
-{
-  capture ${BUILDPACK_HOME}/bin/compile ${BUILD_DIR} ${CACHE_DIR}
-}
-
-release()
-{
-  capture ${BUILDPACK_HOME}/bin/release ${BUILD_DIR}
-}
-
 updateVersion()
 {
   echo "$2" > "test/fixtures/${1}/runtime.txt"
@@ -234,13 +219,18 @@ default_process_types_cleanup() {
   fi
 }
 
+run_in_clean_env() {
+  # Prevent stray environment variables from outside the test runner being exposed to tests.
+  env -i HOME="${HOME}" LANG="${LANG}" PATH="${PATH}" STACK="${STACK}" "$@"
+}
+
 compile() {
   default_process_types_cleanup
   bp_dir=$(mktmpdir)
   compile_dir=$(mktmpdir)
   cp -a $(pwd)/* ${bp_dir}
   cp -a ${bp_dir}/test/fixtures/$1/. ${compile_dir}
-  capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3
+  capture run_in_clean_env ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3
 }
 
 compileDir() {
@@ -252,13 +242,13 @@ compileDir() {
   local env_dir=$3
 
   cp -a $(pwd)/* ${bp_dir}
-  capture ${bp_dir}/bin/compile ${compile_dir} ${cache_dir} ${env_dir}
+  capture run_in_clean_env ${bp_dir}/bin/compile ${compile_dir} ${cache_dir} ${env_dir}
 }
 
 release() {
   bp_dir=$(mktmpdir)
   cp -a $(pwd)/* ${bp_dir}
-  capture ${bp_dir}/bin/release ${bp_dir}/test/fixtures/$1
+  capture run_in_clean_env ${bp_dir}/bin/release ${bp_dir}/test/fixtures/$1
 }
 
 assertFile() {

tests.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-if [[ ! "$STACK" ]]; then
-    echo '$STACK must be set! (heroku-16 | cedar-14)'
-    exit 1
-fi
-
-if [[ "$STACK" == "cedar-14" ]]; then
-    make test-cedar-14
-    exit $?
-fi
-
-if [[ "$STACK" == "heroku-16" ]]; then
-    make test-heroku-16
-    exit $?
-fi
-
-if [[ "$STACK" == "heroku-18" ]]; then
-    make test-heroku-18
-    exit $?
-fi

vendor/pip-pop/pip-diff

@@ -12,9 +12,24 @@ Options:
 """
 import os
 from docopt import docopt
-from pip.req import parse_requirements
+
-from pip.index import PackageFinder
+try:  # pip >= 10
-from pip._vendor.requests import session
+    from pip._internal.req import parse_requirements
+    from pip._internal.download import PipSession as session
+
+    def PackageFinder(find_links, index_urls, session=None):
+        from pip._internal.index import PackageFinder
+        from pip._internal.models.search_scope import SearchScope
+        from pip._internal.models.selection_prefs import SelectionPreferences
+
+        search_scope = SearchScope.create(find_links, index_urls)
+        selection_prefs = SelectionPreferences(allow_yanked=False)
+        return PackageFinder.create(search_scope, selection_prefs, session=session)
+
+except ImportError:  # pip <= 9.0.3
+    from pip.req import parse_requirements
+    from pip.index import PackageFinder
+    from pip._vendor.requests import session
 
 requests = session()
 

vendor/pip-pop/pip-grep

@@ -10,9 +10,25 @@ Options:
 import os
 import sys
 from docopt import docopt
-from pip.req import parse_requirements
+
-from pip.index import PackageFinder
+try:  # pip >= 10
-from pip._vendor.requests import session
+    from pip._internal.req import parse_requirements
+    from pip._internal.download import PipSession as session
+
+    def PackageFinder(find_links, index_urls, session=None):
+        from pip._internal.index import PackageFinder
+        from pip._internal.models.search_scope import SearchScope
+        from pip._internal.models.selection_prefs import SelectionPreferences
+
+        search_scope = SearchScope.create(find_links, index_urls)
+        selection_prefs = SelectionPreferences(allow_yanked=False)
+        return PackageFinder.create(search_scope, selection_prefs, session=session)
+
+except ImportError:  # pip <= 9.0.3
+    from pip.req import parse_requirements
+    from pip.index import PackageFinder
+    from pip._vendor.requests import session
+
 
 requests = session()