Remove deprecated GDAL/GEOS/PROJ support (#1113)

The standalone Geo buildpack offers more modern GDAL/GEOS/PROJ library
versions, and can be used by apps in all languages, not just Python:
https://github.com/heroku/heroku-geo-buildpack

As such the Python buildpack's undocumented built-in support was
deprecated back in April 2020, with a scheduled removal date of
6th October 2020:
https://devcenter.heroku.com/changelog-items/1759
https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-now-deprecated

Metrics show very few builds continuing to use the built-in support.

Apps with the `BUILD_WITH_GEO_LIBRARIES` env var set will now be shown a
warning directing them to the standalone buildpack, as well as apps that
hit GDAL related pip install errors but aren't using the env var.

This also moves us one step closer to being able to remove
the vendored copy of pip-pop (which is partially broken on
newer pip).

Closes @W-7654424@.

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Remove deprecated GDAL/GEOS/PROJ support (#1113).
 - Remove vendored `jq` binary (#1112).
 - Remove redundant Mercurial install step (#1111).
 - Remove support for the Cedar-14 stack (#1110).

Makefile

@@ -12,7 +12,7 @@ STACK_IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
 
 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
-	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/python
+	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/python
 	@shellcheck -x bin/steps/hooks/*
 
 test:

bin/compile

@@ -50,8 +50,6 @@ fi
 # environment variable mechanism (the ENV_DIR).
 S3_BASE_URL="${BUILDPACK_S3_BASE_URL:-"https://heroku-buildpack-python.s3.amazonaws.com"}"
 # This has to be exported since it's used by the geo-libs step which is run in a subshell.
-# TODO: Stop exporting once the geo-libs step is removed or no longer uses `sub_env`.
-export S3_BASE_URL
 
 # Default Python Versions
 # shellcheck source=bin/default_pythons
@@ -137,6 +135,7 @@ export LANG=en_US.UTF-8
 # `~/.heroku/vendor` is an place where the buildpack may stick pre-build binaries for known
 # C dependencies. This section configures Python (GCC, more specifically)
 # and pip to automatically include these paths when building binaries.
+# TODO: Stop adding .heroku/vendor here now that the buildpack no longer vendors anything.
 export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
 export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
 export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
@@ -176,8 +175,6 @@ cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
 cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
 # A plain text file which contains the current sqlite3 version being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-sqlite3-version" .heroku/ &> /dev/null || true
-# Any pre-compiled binaries, provided by the buildpack.
-cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true
 # "editable" installations of code repositories, via pip or pipenv.
 if [[ -d "$CACHE_DIR/.heroku/src" ]]; then
   cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true
@@ -263,16 +260,6 @@ fi
 # shellcheck source=bin/steps/eggpath-fix
 source "$BIN_DIR/steps/eggpath-fix"
 
-# Support for Geo libraries. This is deprecated functionality.
-# It is undocumented.
-# shellcheck source=bin/steps/geo-libs
-sub_env "$BIN_DIR/steps/geo-libs"
-
-# GDAL support.
-# This is part of the Geo support.
-# shellcheck source=bin/steps/gdal
-source "$BIN_DIR/steps/gdal"
-
 # SQLite3 support.
 # This sets up and installs sqlite3 dev headers and the sqlite3 binary but not the
 # libsqlite3-0 library since that exists on the stack image.
@@ -370,7 +357,6 @@ mkdir -p "$CACHE_DIR/.heroku"
 cp -R .heroku/python "$CACHE_DIR/.heroku/"
 cp -R .heroku/python-version "$CACHE_DIR/.heroku/"
 cp -R .heroku/python-stack "$CACHE_DIR/.heroku/" &> /dev/null || true
-cp -R .heroku/vendor "$CACHE_DIR/.heroku/" &> /dev/null || true
 if [[ -d .heroku/src ]]; then
   cp -R .heroku/src "$CACHE_DIR/.heroku/" &> /dev/null || true
 fi

bin/steps/gdal

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-
-# This script serves as the GDAL build step of the
-# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
-# compiler.
-#
-# A [buildpack](https://devcenter.heroku.com/articles/buildpacks) is an
-# adapter between a Python application and Heroku's runtime.
-#
-# This script is invoked by [`bin/compile`](/).
-
-# The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${S3_BASE_URL}/${STACK}/libraries/vendor/gdal.tar.gz"
-
-PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
-
-# Syntax sugar.
-# shellcheck source=bin/utils
-source "$BIN_DIR/utils"
-
-# If GDAL exists within requirements, use vendored gdal.
-if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
-
-  if [[ ! -f ".heroku/vendor/bin/gdalserver" && "${STACK}" == 'heroku-20' ]]; then
-    puts-warn "The buildpack's built-in GDAL functonality is not supported on Heroku-20."
-    puts-warn "Please use this buildpack instead: https://github.com/heroku/heroku-geo-buildpack"
-
-  elif [[ ! -f ".heroku/vendor/bin/gdalserver" ]]; then
-    puts-warn "The vendored GDAL package in the Heroku Python Buildpack now deprecated."
-    puts-warn "To enable GDAL use an alternative buildpack is available here - https://github.com/heroku/heroku-geo-buildpack"
-
-    echo "-----> Noticed GDAL. Bootstrapping gdal."
-    mkdir -p .heroku/vendor
-    # Download and extract cryptography into target vendor directory.
-    curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
-    mcount "steps.vendor.gdal"
-  fi
-
-  GDAL=$(pwd)/vendor
-  export GDAL
-fi

bin/steps/geo-libs

@@ -1,58 +0,0 @@
-#!/usr/bin/env bash
-
-# This script serves as the GDAL build step of the
-# [**Python Buildpack**](https://github.com/heroku/heroku-buildpack-python)
-# compiler.
-#
-# A [buildpack](https://devcenter.heroku.com/articles/buildpacks) is an
-# adapter between a Python application and Heroku's runtime.
-#
-# This script is invoked by [`bin/compile`](/).
-
-# The location of the pre-compiled cryptography binary.
-VENDORED_GDAL="${S3_BASE_URL}/${STACK}/libraries/vendor/gdal.tar.gz"
-VENDORED_GEOS="${S3_BASE_URL}/${STACK}/libraries/vendor/geos.tar.gz"
-VENDORED_PROJ="${S3_BASE_URL}/${STACK}/libraries/vendor/proj.tar.gz"
-
-PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
-
-# Syntax sugar.
-# shellcheck source=bin/utils
-source "$BIN_DIR/utils"
-
-if [[ "$BUILD_WITH_GEO_LIBRARIES" && "${STACK}" == 'heroku-20' ]]; then
-  puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality is not supported on Heroku-20."
-  puts-warn "Please use this buildpack for GDAL, GEOS and PROJ: https://github.com/heroku/heroku-geo-buildpack"
-  puts-warn "To hide this message, unset the BUILD_WITH_GEO_LIBRARIES variable using: heroku config:unset BUILD_WITH_GEO_LIBRARIES"
-elif [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
-  mcount "buildvar.BUILD_WITH_GEO_LIBRARIES"
-
-  puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."
-  puts-warn "An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
-
-  if [ ! -f ".heroku/vendor/bin/proj" ]; then
-    echo "-----> Bootstrapping gdal, geos, proj."
-    mkdir -p .heroku/vendor
-    # Download and extract cryptography into target vendor directory.
-    curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
-    curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null
-    curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null
-
-    mcount "steps.vendor.geo_libs"
-    # Copy libjasper from build image to slug.
-    if [[ "$STACK" == "heroku-16" ]]; then
-      cp /usr/lib/x86_64-linux-gnu/libjasper.so* ".heroku/vendor/lib/."
-      mcount "steps.vendor.libjasper"
-    fi
-
-  fi
-
-  GDAL=$(pwd)/vendor
-  export GDAL
-  # set path for post_compile hooks
-  export GDAL_LIBRARY_PATH="$BUILD_DIR/.heroku/vendor/lib/libgdal.so"
-  export GEOS_LIBRARY_PATH="$BUILD_DIR/.heroku/vendor/lib/libgeos_c.so"
-  # set path for runtime environmeht
-  set_env GDAL_LIBRARY_PATH "/app/.heroku/vendor/lib/libgdal.so"
-  set_env GEOS_LIBRARY_PATH "/app/.heroku/vendor/lib/libgeos_c.so"
-fi

bin/warnings

@@ -46,10 +46,36 @@ six-included() {
   fi
 }
 
+geo-gdal-support() {
+  if [[ -f "${ENV_DIR}/BUILD_WITH_GEO_LIBRARIES" ]]; then
+    mcount "warnings.BUILD_WITH_GEO_LIBRARIES"
+    echo
+    puts-warn "The Python buildpack's BUILD_WITH_GEO_LIBRARIES functonality is no longer supported:"
+    puts-warn "https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-now-deprecated"
+    puts-warn
+    puts-warn "For GDAL, GEOS and PROJ support, use the Geo buildpack alongside the Python buildpack:"
+    puts-warn "https://github.com/heroku/heroku-geo-buildpack"
+    puts-warn
+    puts-warn "To hide this message, unset the BUILD_WITH_GEO_LIBRARIES variable using:"
+    puts-warn "heroku config:unset BUILD_WITH_GEO_LIBRARIES"
+    echo
+    return
+  fi
+  if grep -qi 'Could not find gdal-config' "$WARNINGS_LOG"; then
+    mcount 'warnings.gdal'
+    echo
+    puts-warn "Hello! Package installation failed since the GDAL library was not found."
+    puts-warn "For GDAL, GEOS and PROJ support, use the Geo buildpack alongside the Python buildpack:"
+    puts-warn "https://github.com/heroku/heroku-geo-buildpack"
+    puts-warn "  -- Much Love, Heroku."
+  fi
+}
+
 show-warnings() {
   old-platform
   scipy-included
   distribute-included
   six-included
+  geo-gdal-support
 }
 

builds/libraries/vendor/gdal

@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building gdal…"
-
-VERSION="2.2.1"
-SOURCE_TARBALL="http://download.osgeo.org/gdal/${VERSION}/gdal-${VERSION}.tar.gz"
-
-curl -L $SOURCE_TARBALL | tar zx
-
-pushd "gdal-${VERSION}"
-./configure --prefix=$OUT_PREFIX --enable-static=no  &&
-make
-make install
-
-# Cleanup
-popd

builds/libraries/vendor/geos

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building geos…"
-
-VERSION=3.6.2
-
-SOURCE_TARBALL="http://download.osgeo.org/geos/geos-${VERSION}.tar.bz2"
-
-curl -L $SOURCE_TARBALL | tar xj
-
-pushd "geos-${VERSION}"
-./configure --prefix=$OUT_PREFIX --enable-static=no &&
-make
-make install
-
-# Cleanup
-popd

builds/libraries/vendor/proj

@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/vendor/
-
-OUT_PREFIX=$1
-
-# Use new path, containing autoconf.
-export PATH="/app/.heroku/python/bin/:$PATH"
-hash -r
-
-
-echo "Building gdal…"
-
-VERSION=4.9.3
-SOURCE_TARBALL="http://download.osgeo.org/proj/proj-${VERSION}.tar.gz"
-
-curl -L $SOURCE_TARBALL | tar zx
-
-pushd "proj-${VERSION}"
-./configure --prefix=$OUT_PREFIX --enable-static=no
-make
-make install
-
-# Cleanup
-popd

test/fixtures/gdal/requirements.txt

@@ -0,0 +1 @@
+gdal

test/fixtures/geos/requirements.txt

@@ -1 +0,0 @@
-django

test/run-deps

@@ -17,18 +17,18 @@ testCollectstatic() {
   assertCaptured "collectstatic"
 }
 
-testGEOS() {
+testBuildWithGeoLibrariesWarning() {
   local env_dir="$(mktmpdir)"
   echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
-  compile 'geos' '' "${env_dir}"
-  if [[ $STACK == "heroku-20" ]]; then
-    assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality is not supported on Heroku-20."
-  else
-    assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."
-  fi
-  # This should assertCapturedError on Heroku-20, but the test doesn't actually
-  # install anything that uses GEOS so succeeds (see W-8145375)
-  assertCapturedSuccess
+  compile 'gdal' '' "${env_dir}"
+  assertCaptured " !     The Python buildpack's BUILD_WITH_GEO_LIBRARIES functonality is no longer supported"
+  assertCapturedError
+}
+
+testGDALWarning() {
+  compile 'gdal'
+  assertCaptured " !     Hello! Package installation failed since the GDAL library was not found."
+  assertCapturedError
 }
 
 testNLTK() {

test/run-features

@@ -131,7 +131,6 @@ testHooks() {
     PROFILE_PATH
     PWD
     PYTHONUNBUFFERED
-    S3_BASE_URL
     SHLVL
     SOME_APP_CONFIG_VAR
     STACK