Merge pull request #827 from heroku/fix-python-3-5-7

Fix Python 3.5.7

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Python Buildpack Changelog
 
+# 154 (2019-07-17)
+
+Fix python 3.5.7 formula actually building 3.7.2
+
+# 153 (2019-06-21)
+
+Hotfix for broken heroku-16 deploys
+
+# 152 (2019-04-04)
+
+Python 3.7.3 now available.
+
+# 151 (2019-03-21)
+
+Python 3.5.7 and 3.4.10 now available on all Heroku stacks.
+
+# 150 (2019-03-13)
+
+Python 2.7.16 now available on all Heroku stacks.
+
+# 149 (2019-03-04)
+
+Hotfix for broken Cedar 14 deploys
+
+# 148 (2019-02-21)
+
+No user facing changes, improving internal metrics
+
 # 147 (2019-02-07)
 
 Python 3.7.2 and 3.6.8 now available on all Heroku stacks.

Dockerfile

@@ -3,7 +3,8 @@ FROM heroku/heroku:16-build
 WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
     S3_BUCKET="lang-python" \
-    S3_PREFIX="heroku-16/"
+    S3_PREFIX="heroku-16/" \
+    DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
 

Dockerfile.heroku-18

@@ -3,7 +3,8 @@ FROM heroku/heroku:18-build
 WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
     S3_BUCKET="lang-python" \
-    S3_PREFIX="heroku-18/"
+    S3_PREFIX="heroku-18/" \
+    DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
 

Makefile

@@ -4,7 +4,7 @@
 test: test-heroku-18 test-heroku-16
 
 check:
-	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings
+	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
 	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
 	@shellcheck -x bin/steps/hooks/*
 

bin/compile

@@ -45,16 +45,9 @@ if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then
 fi
 export VENDOR_URL
 
-# Which versions of Python are we using?
+# Default Python Versions
-# These variables are used to specify which versions of Python to install by default,
+# shellcheck source=bin/default_pythons
-# as well as prompt the user to upgrade if they are using an un–supported version.
+source "$BIN_DIR/default_pythons"
-# Note: When 3.7 lands, I recommend switching to LATEST_36 and LATEST_37.
-DEFAULT_PYTHON_VERSION="python-3.6.8"
-LATEST_36="python-3.6.8"
-LATEST_37="python-3.7.2"
-LATEST_35="python-3.5.6"
-LATEST_34="python-3.4.9"
-LATEST_27="python-2.7.15"
 
 # Supported Python Branches
 PY37="python-3.7"
@@ -68,8 +61,7 @@ DEFAULT_PYTHON_STACK="cedar-14"
 # If pip doesn't match this version (the version we install), run the installer.
 PIP_UPDATE="9.0.2"
 
-export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE
+export DEFAULT_PYTHON_STACK PIP_UPDATE
-export LATEST_27 LATEST_36 LATEST_37 LATEST_35 LATEST_34
 export PY37 PY36 PY35 PY27 PY34
 
 # Common Problem Warnings:

bin/default_pythons

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+DEFAULT_PYTHON_VERSION="python-3.6.8"
+LATEST_36="python-3.6.8"
+LATEST_37="python-3.7.3"
+LATEST_35="python-3.5.7"
+LATEST_34="python-3.4.10"
+LATEST_27="python-2.7.16"
+
+export DEFAULT_PYTHON_VERSION LATEST_37 LATEST_36 LATEST_35 LATEST_34 LATEST_27

bin/steps/gdal

@@ -26,9 +26,9 @@ if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
     mkdir -p .heroku/vendor
     # Download and extract cryptography into target vendor directory.
     curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null
+    mcount "steps.vendor.gdal"
   fi
 
   GDAL=$(pwd)/vendor
   export GDAL
 fi
-

bin/steps/geo-libs

@@ -22,6 +22,7 @@ source "$BIN_DIR/utils"
 
 # If GDAL exists within requirements, use vendored gdal.
 if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
+  mcount "buildvar.BUILD_WITH_GEO_LIBRARIES"
 
   if [ ! -f ".heroku/vendor/bin/proj" ]; then
     echo "-----> Bootstrapping gdal, geos, proj."
@@ -31,9 +32,11 @@ if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
     curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null
     curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null
 
+    mcount "steps.vendor.geo_libs"
     # Copy libjasper from build image to slug.
     if [[ "$STACK" == "heroku-16" ]]; then
       cp /usr/lib/x86_64-linux-gnu/libjasper.so* ".heroku/vendor/lib/."
+      mcount "steps.vendor.libjasper"
     fi
 
   fi

bin/steps/mercurial

@@ -4,6 +4,6 @@
 if [[ -f "requirements.txt" ]]; then
 	if (grep -Fiq "hg+" requirements.txt) then
 		/app/.heroku/python/bin/pip install  mercurial | cleanup | indent
+		mcount "steps.mercurial"
 	fi
 fi
-

bin/steps/nltk

@@ -27,6 +27,8 @@ if sp-grep -s nltk; then
         python -m nltk.downloader -d "$BUILD_DIR/.heroku/python/nltk_data" "${nltk_packages[@]}" | indent
         set_env NLTK_DATA "/app/.heroku/python/nltk_data"
 
+        mcount "buildvar.NLTK_PACKAGES_DEFINITION"
+        mcount "steps.nltk"
     else
         puts-warn "'nltk.txt' not found, not downloading any corpora"
         puts-warn "Learn more: https://devcenter.heroku.com/articles/python-nltk"

bin/steps/pylibmc

@@ -29,6 +29,7 @@ if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
     mkdir -p .heroku/vendor
     # Download and extract libmemcached into target vendor directory.
     curl "$VENDORED_MEMCACHED" -s | tar zxv -C .heroku/vendor &> /dev/null
+    mcount "steps.vendor.pylibmc"
   fi
 
   LIBMEMCACHED=$(pwd)/vendor

bin/steps/sqlite3

@@ -4,7 +4,13 @@
 source "$BIN_DIR/utils"
 
 sqlite3_version() {
-  SQLITE3_VERSION=${SQLITE3_VERSION:-$(dpkg -s libsqlite3-0 | grep Version | sed 's/Version: //')}
+  if [ "$STACK" = "cedar-14" ]; then
+    SQLITE3_VERSION="3.8.2-1ubuntu2.2"
+  elif [ "$STACK" = "heroku-16" ]; then
+    SQLITE3_VERSION="3.11.0-1ubuntu1.2"
+  else
+    SQLITE3_VERSION=${SQLITE3_VERSION:-$(dpkg -s libsqlite3-0 | grep Version | sed 's/Version: //')}
+  fi
 
   export SQLITE3_VERSION
 }

builds/runtimes/python-2.7.16

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/2.7.16/Python-2.7.16.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-2.7.16 src
+cd src
+
+./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +

builds/runtimes/python-3.4.10

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.7.2/Python-3.7.2.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.7.2 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.5.7

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.5.7/Python-3.5.7.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.5.7 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.7.3

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.7.3/Python-3.7.3.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.7.3 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

test/fixtures/python2/runtime.txt

@@ -1 +1 @@
-python-2.7.15
+python-2.7.16

test/fixtures/python2_fail/runtime.txt

@@ -1 +1 @@
-python-2.7.16
+python-2.7.99

test/fixtures/python2_warn/runtime.txt

@@ -1 +1 @@
-python-2.7.14
+python-2.7.15

test/fixtures/python3_4_fail/runtime.txt

@@ -1 +1 @@
-python-3.4.10
+python-3.4.99

test/fixtures/python3_5_fail/runtime.txt

@@ -1 +1 @@
-python-3.5.7
+python-3.5.99

test/fixtures/python3_6_fail/runtime.txt

@@ -1 +1 @@
-python-3.6.9
+python-3.6.99

test/fixtures/python3_7_fail/runtime.txt

@@ -1 +1 @@
-python-3.7.3
+python-3.7.99

test/run

@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
 
+# Default Python Versions
+# shellcheck source=bin/default_pythons
+source "bin/default_pythons"
+
 testAirflow() {
   export SLUGIFY_USES_TEXT_UNIDECODE="yes"
   compile "airflow"
@@ -19,13 +23,13 @@ testPipenvLock() {
 
 testPipenvVersion() {
   compile "pipenv-version"
-  assertCaptured "3.6.8"
+  assertCaptured $DEFAULT_PYTHON_VERSION
   assertCapturedSuccess
 }
 
 testPipenvVersion2() {
   compile "pipenv-version2"
-  assertCaptured "2.7.15"
+  assertCaptured $LATEST_27
   assertCapturedSuccess
 }
 testPipenvFullVersion() {
@@ -94,26 +98,27 @@ testPylibmc() {
 }
 
 testPythonDefault() {
+  updateVersion "pythonDefault" $DEFAULT_PYTHON_VERSION
   compile "pythonDefault"
-  assertCaptured "python-3.6.8"
+  assertCaptured $DEFAULT_PYTHON_VERSION
+  assertNotCaptured "security update"
   assertCapturedSuccess
 }
 
 testPython2() {
+    updateVersion "python2" $LATEST_27
+    echo $LATEST_27 > "runtime.txt"
     compile "python2"
-    assertCaptured "python-2.7.15"
+    assertCaptured $LATEST_27
+    assertNotCaptured "security update"
     assertCapturedSuccess
 }
 
 testPython2_warn() {
     compile "python2_warn"
-    if [[ $STACK = "heroku-18" ]]; then
+    assertCaptured "python-2.7.15"
-      assertCapturedError
+    assertCaptured "security update!"
-    else
+    assertCapturedSuccess
-      assertCaptured "python-2.7.14"
-      assertCaptured "security update!"
-      assertCapturedSuccess
-    fi
 }
 
 testPython2_fail() {
@@ -123,9 +128,13 @@ testPython2_fail() {
 }
 
 testPython3_4() {
-  compile "python3_4"
+  if [[ $STACK != "cedar-14" ]]; then
-  assertCaptured "python-3.4.9"
+    updateVersion "python3_4" $LATEST_34
-  assertCapturedSuccess
+    compile "python3_4"
+    assertCaptured $LATEST_34
+    assertNotCaptured "security update"
+    assertCapturedSuccess
+  fi
 }
 
 testPython3_4_warn() {
@@ -146,9 +155,13 @@ testPython3_4_fail() {
 }
 
 testPython3_5() {
-  compile "python3_5"
+  if [[ $STACK != "cedar-14" ]]; then
-  assertCaptured "python-3.5.6"
+    updateVersion "python3_5" $LATEST_35
-  assertCapturedSuccess
+    compile "python3_5"
+    assertCaptured $LATEST_35
+    assertNotCaptured "security update"
+    assertCapturedSuccess
+  fi
 }
 
 testPython3_5_warn() {
@@ -169,8 +182,10 @@ testPython3_5_fail() {
 }
 
 testPython3_6() {
+  updateVersion "python3_6" $LATEST_36
   compile "python3_6"
-  assertCaptured "python-3.6.8"
+  assertCaptured $LATEST_36
+  assertNotCaptured "security update"
   assertCapturedSuccess
 }
 
@@ -188,11 +203,13 @@ testPython3_6_fail() {
 }
 
 testPython3_7() {
+  updateVersion "python3_7" $LATEST_37
   compile "python3_7"
   if [[ $STACK = "cedar-14" ]]; then
     assertCapturedError
   else
-    assertCaptured "python-3.7.2"
+    assertNotCaptured "security update"
+    assertCaptured $LATEST_37
     assertCapturedSuccess
   fi
 }

test/utils

@@ -71,6 +71,11 @@ release()
   capture ${BUILDPACK_HOME}/bin/release ${BUILD_DIR}
 }
 
+updateVersion()
+{
+  echo "$2" > "test/fixtures/${1}/runtime.txt"
+}
+
 assertCapturedEquals()
 {
   assertEquals "$@" "$(cat ${STD_OUT})"