From fcf696b835d7abce1631d841de7ae5fdd5dde45c Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 7 Oct 2020 19:44:33 +0100 Subject: [PATCH] Add support for Heroku-20 (#968) This adds support for the upcoming Heroku-20 stack. The Heroku-20 Dockerfile is identical to that for Heroku-18, other than the base image, and stack-related env var changes. The initial Python versions made available will be those in: https://devcenter.heroku.com/articles/python-support#supported-runtimes https://devcenter.heroku.com/articles/python-support#supported-pypy-runtimes ...minus CPython 2.7, since it's EOL. Which are: * `python-3.6.12` * `python-3.7.9` * `python-3.8.6` * `python-3.9.0` * `pypy2.7-7.3.2` * `pypy3.6-7.3.2` Note: Unlike CPython 2.7, the PyPy 2.7 branch is still supported: https://doc.pypy.org/en/latest/faq.html#how-long-will-pypy-support-python2 In addition, I've generated binaries for each patch release immediately prior to the latest versions (with the exception of 3.9.0, since there isn't one), otherwise it's not possible to run the "out of date Python" warning tests. The binaries were generated using the process here: https://github.com/heroku/heroku-buildpack-python/blob/main/builds/README.md Specifically: ``` make deploy-runtimes STACKS='heroku-20' \ RUNTIMES='python-3.6.11 python-3.6.12 python-3.7.8 python-3.7.9 python-3.8.5 python-3.8.6 python-3.9.0 pypy2.7-7.3.1 pypy2.7-7.3.2 pypy3.6-7.3.1 pypy3.6-7.3.2' \ ENV_FILE=... ``` Binaries for the GDAL/GEOS/PROJ feature have not been generated, since it's deprecated and due for removal shortly: https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-now-deprecated Note: Like the Python 3.9.0 release, this uses the new S3 bucket, so apps will need to be using a recent version of the buildpack in order to build on Heroku-20: https://devcenter.heroku.com/articles/python-support#checking-the-python-buildpack-version Closes @W-7485877@. --- .travis.yml | 4 ++ CHANGELOG.md | 1 + bin/steps/gdal | 5 ++- bin/steps/geo-libs | 7 +++- builds/heroku-20.Dockerfile | 20 ++++++++++ test/fixtures/pipenv-full-version/Pipfile | 2 +- .../fixtures/pipenv-full-version/Pipfile.lock | 17 +-------- test/fixtures/python3_6_warn/runtime.txt | 2 +- test/fixtures/python3_7_warn/runtime.txt | 2 +- test/fixtures/python3_8_warn/runtime.txt | 2 +- test/run-deps | 22 ++++++----- test/run-features | 10 ++++- test/run-versions | 38 +++++++++++++++---- 13 files changed, 92 insertions(+), 40 deletions(-) create mode 100644 builds/heroku-20.Dockerfile diff --git a/.travis.yml b/.travis.yml index 2263dce..63b1589 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,10 @@ env: - STACK=heroku-18 TEST_CMD=test/run-deps - STACK=heroku-18 TEST_CMD=test/run-versions - STACK=heroku-18 TEST_CMD=test/run-features + + - STACK=heroku-20 TEST_CMD=test/run-deps + - STACK=heroku-20 TEST_CMD=test/run-versions + - STACK=heroku-20 TEST_CMD=test/run-features global: - HATCHET_RETRIES=3 - IS_RUNNING_ON_CI=true diff --git a/CHANGELOG.md b/CHANGELOG.md index 2554852..e5b4b0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Add support for Heroku-20 (#968). ## v182 (2020-10-06) diff --git a/bin/steps/gdal b/bin/steps/gdal index 6e3ff9f..6b0fa6f 100755 --- a/bin/steps/gdal +++ b/bin/steps/gdal @@ -21,8 +21,11 @@ source "$BIN_DIR/utils" # If GDAL exists within requirements, use vendored gdal. if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then - if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then + if [[ ! -f ".heroku/vendor/bin/gdalserver" && "${STACK}" == 'heroku-20' ]]; then + puts-warn "The buildpack's built-in GDAL functonality is not supported on Heroku-20." + puts-warn "Please use this buildpack instead: https://github.com/heroku/heroku-geo-buildpack" + elif [[ ! -f ".heroku/vendor/bin/gdalserver" ]]; then puts-warn "The vendored GDAL package in the Heroku Python Buildpack now deprecated." puts-warn "To enable GDAL use an alternative buildpack is available here - https://github.com/heroku/heroku-geo-buildpack" diff --git a/bin/steps/geo-libs b/bin/steps/geo-libs index f3f88da..2e3a85a 100755 --- a/bin/steps/geo-libs +++ b/bin/steps/geo-libs @@ -20,8 +20,11 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH" # shellcheck source=bin/utils source "$BIN_DIR/utils" -# If GDAL exists within requirements, use vendored gdal. -if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then +if [[ "$BUILD_WITH_GEO_LIBRARIES" && "${STACK}" == 'heroku-20' ]]; then + puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality is not supported on Heroku-20." + puts-warn "Please use this buildpack for GDAL, GEOS and PROJ: https://github.com/heroku/heroku-geo-buildpack" + puts-warn "To hide this message, unset the BUILD_WITH_GEO_LIBRARIES variable using: heroku config:unset BUILD_WITH_GEO_LIBRARIES" +elif [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then mcount "buildvar.BUILD_WITH_GEO_LIBRARIES" puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated." diff --git a/builds/heroku-20.Dockerfile b/builds/heroku-20.Dockerfile new file mode 100644 index 0000000..4ee19ed --- /dev/null +++ b/builds/heroku-20.Dockerfile @@ -0,0 +1,20 @@ +FROM heroku/heroku:20-build + +ENV WORKSPACE_DIR="/app/builds" \ + S3_BUCKET="heroku-buildpack-python" \ + S3_PREFIX="heroku-20/" \ + STACK="heroku-20" + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + libsqlite3-dev \ + python3-pip \ + python3-setuptools \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt /app/ +RUN pip3 install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt + +COPY . /app diff --git a/test/fixtures/pipenv-full-version/Pipfile b/test/fixtures/pipenv-full-version/Pipfile index af6f4e4..c91c768 100644 --- a/test/fixtures/pipenv-full-version/Pipfile +++ b/test/fixtures/pipenv-full-version/Pipfile @@ -6,4 +6,4 @@ verify_ssl = true requests = "*" [requires] -python_full_version = "3.6.3" \ No newline at end of file +python_full_version = "3.7.8" diff --git a/test/fixtures/pipenv-full-version/Pipfile.lock b/test/fixtures/pipenv-full-version/Pipfile.lock index 0a758bd..9f364d0 100644 --- a/test/fixtures/pipenv-full-version/Pipfile.lock +++ b/test/fixtures/pipenv-full-version/Pipfile.lock @@ -1,24 +1,11 @@ { "_meta": { "hash": { - "sha256": "22a052f4d1cfe6518b2f236fe45c3208c587a9ab1323bdd390632e27278b541e" - }, - "host-environment-markers": { - "implementation_name": "cpython", - "implementation_version": "3.6.3", - "os_name": "posix", - "platform_machine": "x86_64", - "platform_python_implementation": "CPython", - "platform_release": "16.7.0", - "platform_system": "Darwin", - "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64", - "python_full_version": "3.6.3", - "python_version": "3.6", - "sys_platform": "darwin" + "sha256": "8a36860f0f9cb55716222098062cea5c5e0f8127cafb9d0c694de327bac9fbc0" }, "pipfile-spec": 6, "requires": { - "python_full_version": "3.6.3" + "python_full_version": "3.7.8" }, "sources": [ { diff --git a/test/fixtures/python3_6_warn/runtime.txt b/test/fixtures/python3_6_warn/runtime.txt index 34b35b7..a06518f 100644 --- a/test/fixtures/python3_6_warn/runtime.txt +++ b/test/fixtures/python3_6_warn/runtime.txt @@ -1 +1 @@ -python-3.6.7 +python-3.6.11 diff --git a/test/fixtures/python3_7_warn/runtime.txt b/test/fixtures/python3_7_warn/runtime.txt index 4255f73..32a8d83 100644 --- a/test/fixtures/python3_7_warn/runtime.txt +++ b/test/fixtures/python3_7_warn/runtime.txt @@ -1 +1 @@ -python-3.7.1 +python-3.7.8 diff --git a/test/fixtures/python3_8_warn/runtime.txt b/test/fixtures/python3_8_warn/runtime.txt index 73b1cf8..43b47fb 100644 --- a/test/fixtures/python3_8_warn/runtime.txt +++ b/test/fixtures/python3_8_warn/runtime.txt @@ -1 +1 @@ -python-3.8.0 +python-3.8.5 diff --git a/test/run-deps b/test/run-deps index 8b69f2a..beb87f6 100755 --- a/test/run-deps +++ b/test/run-deps @@ -21,16 +21,13 @@ testGEOS() { local env_dir="$(mktmpdir)" echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES" compile 'geos' '' "${env_dir}" - assertCaptured "geos" - assertCapturedSuccess -} - -testGEOSDeprecation() { - local env_dir="$(mktmpdir)" - echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES" - compile 'geos' '' "${env_dir}" - assertCaptured " ! The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated. - ! An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack" + if [[ $STACK == "heroku-20" ]]; then + assertCaptured " ! The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality is not supported on Heroku-20." + else + assertCaptured " ! The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated." + fi + # This should assertCapturedError on Heroku-20, but the test doesn't actually + # install anything that uses GEOS so succeeds (see W-8145375) assertCapturedSuccess } @@ -55,6 +52,11 @@ testPsycopg2() { } testPysqlite() { + # pysqlite does not support Python 3 (since the sqlite3 stdlib can be used there), + # so we have to test with Python 2, which we've not made available for Heroku-20. + if [[ $STACK == "heroku-20" ]]; then + return + fi compile "pysqlite" assertCaptured "pysqlite" assertCapturedSuccess diff --git a/test/run-features b/test/run-features index f67da7b..a873788 100755 --- a/test/run-features +++ b/test/run-features @@ -68,6 +68,10 @@ testPipenvPythonVersion3_6() { } testPipenvPythonVersion2_7() { + # Python 2.7 is EOL, so it has not been built for Heroku-20. + if [[ $STACK == "heroku-20" ]]; then + return + fi compile "pipenv-version2" assertCaptured "Installing ${LATEST_27}" # Can't use `assertCapturedSuccess` since stderr contains: @@ -76,8 +80,12 @@ testPipenvPythonVersion2_7() { } testPipenvPythonFullVersion() { + # Python 3.7+ requires newer libssl than is present on Cedar-14. + if [[ "${STACK}" = "cedar-14" ]]; then + return + fi compile "pipenv-full-version" - assertCaptured "3.6.3" + assertCaptured "3.7.8" # Can't use `assertCapturedSuccess` since stderr contains: # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941) assertCapturedSuccessWithStdErr diff --git a/test/run-versions b/test/run-versions index 15b6e99..63d2751 100755 --- a/test/run-versions +++ b/test/run-versions @@ -14,6 +14,10 @@ testPythonVersionUnspecified() { } testPython2_7() { + # Python 2.7 is EOL, so it has not been built for Heroku-20. + if [[ $STACK == "heroku-20" ]]; then + return + fi compile "python2" assertCaptured "Installing ${LATEST_27}" assertCaptured "python-2-7-eol-faq"; @@ -24,6 +28,10 @@ testPython2_7() { } testPython2_7_warn() { + # Python 2.7 is EOL, so it has not been built for Heroku-20. + if [[ $STACK == "heroku-20" ]]; then + return + fi compile "python2_warn" assertCaptured "Installing python-2.7.15" assertCaptured "python-2-7-eol-faq"; @@ -33,6 +41,10 @@ testPython2_7_warn() { } testPython3_4() { + # Python 3.4 is EOL, so it has not been built for Heroku-20. + if [[ $STACK == "heroku-20" ]]; then + return + fi compile "python3_4" assertCaptured "Installing ${LATEST_34}" assertNotCaptured "security update" @@ -44,6 +56,10 @@ testPython3_4() { } testPython3_4_warn() { + # Python 3.4 is EOL, so it has not been built for Heroku-20. + if [[ $STACK == "heroku-20" ]]; then + return + fi compile "python3_4_warn" assertCaptured "Installing python-3.4.9" assertCaptured "security update!" @@ -54,6 +70,10 @@ testPython3_4_warn() { } testPython3_5() { + # Python 3.5 is EOL, so it has not been built for Heroku-20. + if [[ $STACK == "heroku-20" ]]; then + return + fi compile "python3_5" assertCaptured "Installing ${LATEST_35}" assertNotCaptured "security update" @@ -63,6 +83,10 @@ testPython3_5() { } testPython3_5_warn() { + # Python 3.5 is EOL, so it has not been built for Heroku-20. + if [[ $STACK == "heroku-20" ]]; then + return + fi compile "python3_5_warn" assertCaptured "Installing python-3.5.6" assertCaptured "security update!" @@ -81,7 +105,7 @@ testPython3_6() { testPython3_6_warn() { compile "python3_6_warn" - assertCaptured "Installing python-3.6.7" + assertCaptured "Installing python-3.6.11" assertCaptured "security update!" assertCaptured "${LATEST_36}" assertCapturedSuccess @@ -106,7 +130,7 @@ testPython3_7_warn() { return fi compile "python3_7_warn" - assertCaptured "Installing python-3.7.1" + assertCaptured "Installing python-3.7.8" assertCaptured "security update!" assertCaptured "${LATEST_37}" assertCapturedSuccess @@ -131,7 +155,7 @@ testPython3_8_warn() { return fi compile "python3_8_warn" - assertCaptured "Installing python-3.8.0" + assertCaptured "Installing python-3.8.5" assertCaptured "security update!" assertCaptured "${LATEST_38}" assertCapturedSuccess @@ -208,7 +232,7 @@ testPypy2_7_warn() { testStickyPythonVersion() { local cache_dir="$(mktmpdir)" compile "python3_6_warn" "$cache_dir" - assertCaptured "Installing python-3.6.7" + assertCaptured "Installing python-3.6.11" assertCapturedSuccess compile "python_version_unspecified" "$cache_dir" assertNotCaptured "Installing python" @@ -217,16 +241,16 @@ testStickyPythonVersion() { # Whilst this file seems like an implementation detail (so something that should # not be tested), we must guarantee the filename remains consistent for backwards # compatibility across buildpack versions for already-built apps. - assertFile "python-3.6.7" ".heroku/python-version" + assertFile "python-3.6.11" ".heroku/python-version" } testPythonVersionChange() { local cache_dir="$(mktmpdir)" compile "python3_6_warn" "$cache_dir" - assertCaptured "Installing python-3.6.7" + assertCaptured "Installing python-3.6.11" assertCapturedSuccess compile "python3_6" "$cache_dir" - assertCaptured "Found python-3.6.7, removing" + assertCaptured "Found python-3.6.11, removing" assertCapturedSuccess }