Merge pull request #736 from heroku/v-138

v 138 changelog

.travis.yml

@@ -9,6 +9,11 @@ jobs:
         - PATH="/tmp/shellcheck-latest:$PATH"
       script: make check
 
+    - stage: "Stack Tests"
+      services: docker
+      env: STACK=heroku-18
+      script: ./tests.sh
+
     - stage: "Stack Tests"
       services: docker
       env: STACK=heroku-16

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Python Buildpack Changelog
 
+# 138
+
+Use stack image SQLite3 instead of vendoring
+
+# 137
+
+Prevent 3.7.0 from appearing as unsupported in buildpack messaging.
+
 # 136
 
 Upgrade to 3.6.6 and support 3.7.0 on all runtimes.

Dockerfile.heroku-18

@@ -0,0 +1,13 @@
+FROM heroku/heroku:18-build
+
+WORKDIR /app
+ENV WORKSPACE_DIR="/app/builds" \
+    S3_BUCKET="lang-python" \
+    S3_PREFIX="heroku-18/"
+
+RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/
+RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+
+COPY . /app

Makefile

@@ -18,6 +18,11 @@ test-heroku-16:
 	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
 	@echo ""
 
+test-heroku-18:
+	@echo "Running tests in docker (heroku-18)..."
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@echo ""
+
 buildenv-heroku-16:
 	@echo "Creating build environment (heroku-16)..."
 	@echo

bin/compile

@@ -50,7 +50,8 @@ export VENDOR_URL
 # as well as prompt the user to upgrade if they are using an un–supported version.
 # Note: When 3.7 lands, I recommend switching to LATEST_36 and LATEST_37.
 DEFAULT_PYTHON_VERSION="python-3.6.6"
-LATEST_3="python-3.6.6"
+LATEST_36="python-3.6.6"
+LATEST_37="python-3.7.0"
 LATEST_2="python-2.7.15"
 
 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
@@ -58,7 +59,7 @@ DEFAULT_PYTHON_STACK="cedar-14"
 # If pip doesn't match this version (the version we install), run the installer.
 PIP_UPDATE="9.0.2"
 
-export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE LATEST_2 LATEST_3
+export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE LATEST_2 LATEST_36 LATEST_37
 
 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.
@@ -159,6 +160,8 @@ cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true
 cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
 # A plain text file which contains the current python version being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
+# A plain text file which contains the current sqlite3 version being used (used for cache busting).
+cp -R "$CACHE_DIR/.heroku/python-sqlite3-version" .heroku/ &> /dev/null || true
 # Any pre-compiled binaries, provided by the buildpack.
 cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true
 # "editable" installations of code repositories, via pip or pipenv.
@@ -273,6 +276,16 @@ sub_env "$BIN_DIR/steps/geo-libs"
 # shellcheck source=bin/steps/gdal
 source "$BIN_DIR/steps/gdal"
 
+# SQLite3 support.
+# This sets up and installs sqlite3 dev headers and the sqlite3 binary but not the
+# libsqlite3-0 library since that exists on the stack image.
+# Note: This only applies to Python 2.7.15+ and Python 3.6.6+
+(( start=$(nowms) ))
+# shellcheck source=bin/steps/sqlite3
+source "$BIN_DIR/steps/sqlite3"
+buildpack_sqlite3_install
+mtime "sqlite3.install.time" "${start}"
+
 # pip install
 # -----------
 

bin/steps/pipenv-python-version

@@ -22,7 +22,10 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
                     echo "$LATEST_2" > "$BUILD_DIR/runtime.txt"
                 fi
                 if [ "$PYTHON" = 3.6 ]; then
-                    echo "$LATEST_3" > "$BUILD_DIR/runtime.txt"
+                    echo "$LATEST_36" > "$BUILD_DIR/runtime.txt"
+                fi
+                if [ "$PYTHON" = 3.7 ]; then
+                    echo "$LATEST_37" > "$BUILD_DIR/runtime.txt"
                 fi
             fi
 

bin/steps/python

@@ -14,17 +14,30 @@ if [[ $PYTHON_VERSION =~ ^python-2 ]]; then
     echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
   fi
 else
-  if [[ "$PYTHON_VERSION" != "$LATEST_3" ]]; then
+  if [[ $PYTHON_VERSION =~ ^python-3.7 ]] && [[ "$PYTHON_VERSION" != "$LATEST_37" ]]; then
-    puts-warn "The latest version of Python 3 is $LATEST_3 (you are using $PYTHON_VERSION, which is unsupported)."
+    puts-warn "The latest version of Python 3.7 is $LATEST_37 (you are using $PYTHON_VERSION, which is unsupported)."
-    puts-warn "We recommend upgrading by specifying the latest version ($LATEST_3)."
+    puts-warn "We recommend upgrading by specifying the latest version ($LATEST_37)."
     echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+  else
+    if [[ "$PYTHON_VERSION" != "$LATEST_36" ]]; then
+      puts-warn "The latest version of Python 3.6 is $LATEST_36 (you are using $PYTHON_VERSION, which is unsupported)."
+      puts-warn "We recommend upgrading by specifying the latest version ($LATEST_36)."
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
   fi
-
 fi
 
 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
     puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache"
-    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
+    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
+fi
+
+# need to clear the cache for first time installing SQLite3,
+# since the version is changing and could lead to runtime errors
+# with compiled extensions.
+if [ -d .heroku/python ] && [ ! -f .heroku/python-sqlite3-version ] && python_sqlite3_check "$PYTHON_VERSION"; then
+  puts-step "Need to update SQLite3, clearing cache"
+  rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
 fi
 
 if [ -f .heroku/python-version ]; then

bin/steps/sqlite3

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
+
+sqlite3_version() {
+  SQLITE3_VERSION=${SQLITE3_VERSION:-$(dpkg -s libsqlite3-0 | grep Version | sed 's/Version: //')}
+
+  export SQLITE3_VERSION
+}
+
+sqlite3_install() {
+  HEROKU_PYTHON_DIR="$1"
+  SQLITE3_VERSION="$2"
+  HEADERS_ONLY="$3"
+
+  mkdir -p "$HEROKU_PYTHON_DIR"
+
+  APT_CACHE_DIR="$HEROKU_PYTHON_DIR/apt/cache"
+  APT_STATE_DIR="$HEROKU_PYTHON_DIR/apt/state"
+
+  mkdir -p "$APT_CACHE_DIR/archives/partial"
+  mkdir -p "$APT_STATE_DIR/lists/partial"
+
+  APT_OPTIONS="-o debug::nolocking=true"
+  APT_OPTIONS="$APT_OPTIONS -o dir::cache=$APT_CACHE_DIR"
+  APT_OPTIONS="$APT_OPTIONS -o dir::state=$APT_STATE_DIR"
+  APT_OPTIONS="$APT_OPTIONS -o dir::etc::sourcelist=/etc/apt/sources.list"
+
+  apt-get $APT_OPTIONS update > /dev/null 2>&1
+  if [ -z "$HEADERS_ONLY" ]; then
+    apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION" sqlite3="$SQLITE3_VERSION" > /dev/null 2>&1
+  else
+    apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION"
+  fi
+
+  find "$APT_CACHE_DIR/archives/" -name "*.deb" -exec dpkg -x {} "$HEROKU_PYTHON_DIR/sqlite3/" \;
+
+  mkdir -p "$HEROKU_PYTHON_DIR/include"
+  mkdir -p "$HEROKU_PYTHON_DIR/lib"
+
+  # remove old sqlite3 libraries/binaries
+  find "$HEROKU_PYTHON_DIR/include/" -name "sqlite3*.h" -exec rm -f {} \;
+  find "$HEROKU_PYTHON_DIR/lib/" -name "libsqlite3.*" -exec rm -f {} \;
+  rm -f "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc"
+  rm -f "$HEROKU_PYTHON_DIR/bin/sqlite3"
+
+  # copy over sqlite3 headers & bins and setup linking against the stack image library
+  mv "$HEROKU_PYTHON_DIR/sqlite3/usr/include/"* "$HEROKU_PYTHON_DIR/include/"
+  mv "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu"/libsqlite3.*a "$HEROKU_PYTHON_DIR/lib/"
+  mkdir -p "$HEROKU_PYTHON_DIR/lib/pkgconfig"
+  # set the right prefix/lib directories
+  sed -e 's/prefix=\/usr/prefix=\/app\/.heroku\/python/' -e 's/\/x86_64-linux-gnu//' "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/pkgconfig/sqlite3.pc" > "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc"
+  # need to point the libsqlite3.so to the stack image library for /usr/bin/ld -lsqlite3
+  SQLITE3_LIBFILE="/usr/lib/x86_64-linux-gnu/$(readlink -n "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/libsqlite3.so")"
+  ln -s "$SQLITE3_LIBFILE" "$HEROKU_PYTHON_DIR/lib/libsqlite3.so"
+  if [ -z "$HEADERS_ONLY" ]; then
+    mv "$HEROKU_PYTHON_DIR/sqlite3/usr/bin"/* "$HEROKU_PYTHON_DIR/bin/"
+  fi
+
+  # cleanup
+  rm -rf "$HEROKU_PYTHON_DIR/sqlite3/"
+  rm -rf "$HEROKU_PYTHON_DIR/apt/"
+}
+
+buildpack_sqlite3_install() {
+  sqlite3_version
+  HEROKU_PYTHON_DIR="$BUILD_DIR/.heroku/python"
+
+  SQLITE3_VERSION_FILE="$BUILD_DIR/.heroku/python-sqlite3-version"
+  if [ -f "$SQLITE3_VERSION_FILE" ]; then
+    INSTALLED_SQLITE3_VERSION=$(cat "$SQLITE3_VERSION_FILE")
+  fi
+
+  # python version check
+  if python_sqlite3_check "$PYTHON_VERSION"; then
+    # only install if the sqlite3 version has changed
+    if [ "$INSTALLED_SQLITE3_VERSION" != "$SQLITE3_VERSION" ]; then
+      puts-step "Installing SQLite3"
+      sqlite3_install "$BUILD_DIR/.heroku/python" "$SQLITE3_VERSION"
+
+      # save version installed
+      mkdir -p "$CACHE_DIR/.heroku/"
+      echo "$SQLITE3_VERSION" > "$CACHE_DIR/.heroku/python-sqlite3-version"
+    fi
+  fi
+}

bin/utils

@@ -58,3 +58,41 @@ measure-size() {
   echo "$(du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')"
 }
 
+# Python version operator >
+version_gt() {
+  test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1";
+}
+
+# Python verison operator >=
+version_gte() {
+  if [ "$1" == "$2" ]; then
+    return 0
+  fi
+
+  version_gt "$1" "$2"
+}
+
+# Check if Python 2
+python2_check() {
+  VERSION="$1"
+
+  version_gte "$VERSION" "python-2.7.0" && version_gt "python-3.0.0" "$VERSION"
+}
+
+# Check if Python 3
+python3_check() {
+  VERSION="$1"
+
+  version_gte "$VERSION" "python-3.0.0" && version_gt "python-4.0.0" "$VERSION"
+}
+
+# Check if Python version needs to install SQLite3
+python_sqlite3_check() {
+  VERSION="$1"
+  MIN_PYTHON_3="python-3.6.6"
+  MIN_PYTHON_2="python-2.7.15"
+
+  ( python2_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_2" ) \
+    || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" ) \
+    || ( version_gte "$VERSION" "3.7.0" )
+}

builds/runtimes/python-2.7.15

@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
 
 OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
 
 echo "Building Python…"
 SOURCE_TARBALL='https://python.org/ftp/python/2.7.15/Python-2.7.15.tgz'

builds/runtimes/python-3.6.6

@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
 
 OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
 
 echo "Building Python…"
 SOURCE_TARBALL='https://python.org/ftp/python/3.6.6/Python-3.6.6.tgz'

builds/runtimes/python-3.7.0

@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
 
 OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
 
 echo "Building Python…"
 SOURCE_TARBALL='https://python.org/ftp/python/3.7.0/Python-3.7.0.tgz'

etc/publish.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+
+BP_NAME=${1:-"heroku/python"}
+
+curVersion=$(heroku buildpacks:versions "$BP_NAME" | awk 'FNR == 3 { print $1 }')
+newVersion="v$((curVersion + 1))"
+
+read -p "Deploy as version: $newVersion [y/n]? " choice
+case "$choice" in
+  y|Y ) echo "";;
+  n|N ) exit 0;;
+  * ) exit 1;;
+esac
+
+originMaster=$(git rev-parse origin/master)
+echo "Tagging commit $originMaster with $newVersion... "
+git tag "$newVersion" "${originMaster:?}"
+git push origin refs/tags/$newVersion
+
+heroku buildpacks:publish "$BP_NAME" "$newVersion"
+
+if [ $(git tag | grep -q previous-version) ]; then
+    echo "Updating previous-version tag"
+    git tag -d previous-version
+    git push origin :previous-version
+    git tag previous-version latest-version
+fi
+if [ $(git tag | grep -q latest-version) ]; then
+    echo "Updating latest-version tag"
+    git tag -d latest-version
+    git push origin :latest-version
+    git tag latest-version "${originMaster:?}"
+    git push --tags
+fi
+
+echo "Done."

test/fixtures/nltk/runtime.txt

@@ -1 +1 @@
-python-2.7.13
+python-3.6.6

test/run

@@ -39,6 +39,11 @@ testGEOS() {
 }
 
 testNLTK() {
+  # NOTE: This is a RuntimeWarning emitted by Python 3's runpy.py script
+  # which is what is used when you call `python -m <module>`. This is due to
+  # how nltk imports things. It's not actually an error, but it would probably
+  # be bad to silence in Production.
+  export PYTHONWARNINGS="ignore::RuntimeWarning"
   compile "nltk"
   assertCaptured "Downloading NLTK packages: city_database stopwords"
   assertCapturedSuccess
@@ -76,9 +81,18 @@ testPylibmc() {
 }
 
 testPython2() {
-  compile "python2"
+  if [[ "$STACK" == "heroku-16" ]] || [[ "$STACK" == "cedar-14" ]]; then
-  assertCaptured "python-2.7.15"
+    compile "python2"
-  assertCapturedSuccess
+    assertCaptured "python-2.7.15"
+    assertCapturedSuccess
+  fi
+}
+
+testNoPython2() {
+  if [[ "$STACK" == "heroku-18" ]]; then
+    compile "python2"
+    assertCapturedError
+  fi
 }
 
 testPython3() {

tests.sh

@@ -13,4 +13,9 @@ fi
 if [[ "$STACK" == "heroku-16" ]]; then
     make test-heroku-16
     exit $?
-fi
+fi
+
+if [[ "$STACK" == "heroku-18" ]]; then
+    make test-heroku-18
+    exit $?
+fi