Merge pull request #781 from heroku/update-hatchet-setup

Update to reflect Hatchet updates

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,8 @@
+<!-- Hi and welcome to the Heroku Python buildpack repository!
+
+If you meant to open a PR against a fork instead of upstream, please adjust the base branch:
+https://help.github.com/articles/changing-the-base-branch-of-a-pull-request/
+
+Otherwise thank you in advance for your Pull Request - just remember to
+include as much information as possible to help the reviewers :-)
+-->

.gitignore

@@ -3,3 +3,7 @@ site
 .DS_Store
 
 /.envrc
+repos/*
+
+#Venv
+buildpack/*

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+repos:
+-   repo: git://github.com/detailyang/pre-commit-shell
+    rev: 1.0.4
+    hooks:
+    - id: shell-lint

.travis.yml

@@ -1,20 +1,38 @@
-language: bash
+language: ruby
 dist: trusty
+sudo: required
+rvm:
+- 2.4.4
+before_script:
+  - gem install bundler -v 1.16.2
+  - bundle exec hatchet ci:setup
 jobs:
   include:
-    - stage: "Bash linting (shellcheck)"
-      sudo: false
-      before_install:
-        - wget -c https://goo.gl/ZzKHFv -O - | tar -xvJ -C /tmp/
-        - PATH="/tmp/shellcheck-latest:$PATH"
-      script: make check
-
-    - stage: "Stack Tests"
-      services: docker
-      env: STACK=heroku-16
-      script: ./tests.sh
-
-    - stage: "Stack Tests"
-      services: docker
-      env: STACK=cedar-14
-      script: ./tests.sh
+  - stage: Bash linting (shellcheck)
+    sudo: false
+    before_install:
+    - wget -c https://goo.gl/ZzKHFv -O - | tar -xvJ -C /tmp/
+    - PATH="/tmp/shellcheck-latest:$PATH"
+    script: make check
+  - stage: Stack Unit Tests
+    services: docker
+    env: STACK=heroku-18
+    script: "./tests.sh"
+  - stage: Stack Unit Tests
+    services: docker
+    env: STACK=heroku-16
+    script: "./tests.sh"
+  - stage: Stack Unit Tests
+    services: docker
+    env: STACK=cedar-14
+    script: "./tests.sh"
+  - stage: Hatchet Integration
+    script: "bundle exec rspec"
+env:
+  global:
+  - HATCHET_RETRIES=3
+  - IS_RUNNING_ON_CI=true
+  - HATCHET_APP_LIMIT=5
+  - HATCHET_DEPLOY_STRATEGY=git
+  - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
+  - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=

CHANGELOG.md

@@ -1,14 +1,44 @@
 # Python Buildpack Changelog
 
-# 136
+# 145 (2018-11-08)
+
+Python 3.7.1, 3.6.7, 3.5.6 and 3.4.9 now available on all Heroku stacks.
+
+# 144 (2018-10-10)
+
+Switch to cautious upgrade for Pipenv install to ensure the pinned pip version
+is used with Pipenv
+
+# 143 (2018-10-09)
+
+Add support for detecting SLUGIFY_USES_TEXT_UNIDECODE, which is required to
+install Apache Airflow version 1.10 or higher.
+
+# 142 (2018-10-08)
+
+Improvements to Python install messaging
+
+# 139, 140, 141
+
+No user-facing changes, documenting for version clarity
+
+# 138 (2018-08-01)
+
+Use stack image SQLite3 instead of vendoring
+
+# 137 (2018-07-17)
+
+Prevent 3.7.0 from appearing as unsupported in buildpack messaging.
+
+# 136 (2018-06-28)
 
 Upgrade to 3.6.6 and support 3.7.0 on all runtimes.
 
-# 135
+# 135 (2018-05-29)
 
 Upgrade Pipenv to v2018.5.18.
 
-# 134
+# 134 (2018-05-02)
 
 Default to 3.6.5, bugfixes.
 

Dockerfile.heroku-18

@@ -0,0 +1,13 @@
+FROM heroku/heroku:18-build
+
+WORKDIR /app
+ENV WORKSPACE_DIR="/app/builds" \
+    S3_BUCKET="lang-python" \
+    S3_PREFIX="heroku-18/"
+
+RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/
+RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+
+COPY . /app

Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+gem "rspec"
+gem "heroku_hatchet"
+gem "rspec-retry"
+gem "rake"

Gemfile.lock

@@ -0,0 +1,71 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    concurrent-ruby (1.1.3)
+    diff-lcs (1.3)
+    erubis (2.7.0)
+    excon (0.62.0)
+    heroics (0.0.25)
+      erubis (~> 2.0)
+      excon
+      moneta
+      multi_json (>= 1.9.2)
+    heroku_hatchet (4.0.6)
+      excon (~> 0)
+      minitest-retry (~> 0.1.9)
+      platform-api (~> 2)
+      repl_runner (~> 0.0.3)
+      rrrretry (~> 1)
+      thor (~> 0)
+      threaded (~> 0)
+    i18n (1.1.1)
+      concurrent-ruby (~> 1.0)
+    minitest (5.11.3)
+    minitest-retry (0.1.9)
+      minitest (>= 5.0)
+    moneta (1.0.0)
+    multi_json (1.13.1)
+    platform-api (2.2.0)
+      heroics (~> 0.0.25)
+      moneta (~> 1.0.0)
+    rake (12.3.1)
+    repl_runner (0.0.3)
+      activesupport
+    rrrretry (1.0.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-retry (0.6.1)
+      rspec-core (> 3.3)
+    rspec-support (3.8.0)
+    thor (0.20.3)
+    thread_safe (0.3.6)
+    threaded (0.0.4)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  heroku_hatchet
+  rake
+  rspec
+  rspec-retry
+
+BUNDLED WITH
+   1.16.3

Makefile

@@ -18,6 +18,11 @@ test-heroku-16:
 	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
 	@echo ""
 
+test-heroku-18:
+	@echo "Running tests in docker (heroku-18)..."
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@echo ""
+
 buildenv-heroku-16:
 	@echo "Creating build environment (heroku-16)..."
 	@echo

Rakefile

@@ -0,0 +1 @@
+require 'hatchet/tasks'

bin/compile

@@ -50,15 +50,27 @@ export VENDOR_URL
 # as well as prompt the user to upgrade if they are using an un–supported version.
 # Note: When 3.7 lands, I recommend switching to LATEST_36 and LATEST_37.
 DEFAULT_PYTHON_VERSION="python-3.6.6"
-LATEST_3="python-3.6.6"
-LATEST_2="python-2.7.15"
+LATEST_36="python-3.6.6"
+LATEST_37="python-3.7.0"
+LATEST_35="python-3.5.6"
+LATEST_34="python-3.4.9"
+LATEST_27="python-2.7.15"
+
+# Supported Python Branches
+PY37="python-3.7"
+PY36="python-3.6"
+PY35="python-3.5"
+PY34="python-3.4"
+PY27="python-2.7"
 
 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
 # If pip doesn't match this version (the version we install), run the installer.
 PIP_UPDATE="9.0.2"
 
-export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE LATEST_2 LATEST_3
+export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE
+export LATEST_27 LATEST_36 LATEST_37 LATEST_35 LATEST_34
+export PY37 PY36 PY35 PY27 PY34
 
 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.
@@ -159,6 +171,8 @@ cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true
 cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
 # A plain text file which contains the current python version being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
+# A plain text file which contains the current sqlite3 version being used (used for cache busting).
+cp -R "$CACHE_DIR/.heroku/python-sqlite3-version" .heroku/ &> /dev/null || true
 # Any pre-compiled binaries, provided by the buildpack.
 cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true
 # "editable" installations of code repositories, via pip or pipenv.
@@ -273,6 +287,16 @@ sub_env "$BIN_DIR/steps/geo-libs"
 # shellcheck source=bin/steps/gdal
 source "$BIN_DIR/steps/gdal"
 
+# SQLite3 support.
+# This sets up and installs sqlite3 dev headers and the sqlite3 binary but not the
+# libsqlite3-0 library since that exists on the stack image.
+# Note: This only applies to Python 2.7.15+ and Python 3.6.6+
+(( start=$(nowms) ))
+# shellcheck source=bin/steps/sqlite3
+source "$BIN_DIR/steps/sqlite3"
+buildpack_sqlite3_install
+mtime "sqlite3.install.time" "${start}"
+
 # pip install
 # -----------
 

bin/steps/pip-install

@@ -8,10 +8,24 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     # Install dependencies with Pip.
     puts-step "Installing requirements with pip"
 
-    # Set PIP_EXTRA_INDEX_URL
-    if [[ -r $ENV_DIR/PIP_EXTRA_INDEX_URL ]]; then
+    # Set Pip env vars
+    # This reads certain environment variables set on the Heroku app config
+    # and makes them accessible to the pip install process.
+    #
+    # PIP_EXTRA_INDEX_URL allows for an alternate pypi URL to be used.
+    if [[ -r "$ENV_DIR/PIP_EXTRA_INDEX_URL" ]]; then
         PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")"
         export PIP_EXTRA_INDEX_URL
+        mcount "buildvar.PIP_EXTRA_INDEX_URL"
+    fi
+
+    set +e
+
+    # Set SLUGIFY_USES_TEXT_UNIDECODE, required for Airflow versions >=1.10
+    if [[ -r "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE" ]]; then
+        SLUGIFY_USES_TEXT_UNIDECODE="$(cat "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE")"
+        export SLUGIFY_USES_TEXT_UNIDECODE
+        mcount "buildvar.SLUGIFY_USES_TEXT_UNIDECODE"
     fi
 
     set +e

bin/steps/pipenv

@@ -20,7 +20,6 @@ if [[ -f Pipfile.lock ]]; then
                 export SKIP_PIPENV_INSTALL=1
                 export SKIP_PIP_INSTALL=1
             fi
-
         fi
     fi
 fi
@@ -35,16 +34,31 @@ if [ ! "$SKIP_PIPENV_INSTALL" ]; then
         # Skip pip install, later.
         export SKIP_PIP_INSTALL=1
 
-        # Set PIP_EXTRA_INDEX_URL
-        if [[ -r $ENV_DIR/PIP_EXTRA_INDEX_URL ]]; then
+        # Set Pip env vars
+        # This reads certain environment variables set on the Heroku app config
+        # and makes them accessible to the pip install process.
+        #
+        # PIP_EXTRA_INDEX_URL allows for an alternate pypi URL to be used.
+        if [[ -r "$ENV_DIR/PIP_EXTRA_INDEX_URL" ]]; then
             PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")"
             export PIP_EXTRA_INDEX_URL
+            mcount "buildvar.PIP_EXTRA_INDEX_URL"
+        fi
+
+        # Set SLUGIFY_USES_TEXT_UNIDECODE, required for Airflow versions >=1.10
+        if [[ -r "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE" ]]; then
+            SLUGIFY_USES_TEXT_UNIDECODE="$(cat "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE")"
+            export SLUGIFY_USES_TEXT_UNIDECODE
+            mcount "buildvar.SLUGIFY_USES_TEXT_UNIDECODE"
         fi
 
         export PIPENV_VERSION="2018.5.18"
 
         # Install pipenv.
-        /app/.heroku/python/bin/pip install pipenv==$PIPENV_VERSION --upgrade &> /dev/null
+        # Due to weird old pip behavior and pipenv behavior, pipenv upgrades pip
+        # to latest if only --upgrade is specified. Specify upgrade strategy to
+        # avoid this eager behavior.
+        /app/.heroku/python/bin/pip install pipenv==$PIPENV_VERSION --upgrade --upgrade-strategy only-if-needed &> /dev/null
 
         # Install the dependencies.
         if [[ ! -f Pipfile.lock ]]; then

bin/steps/pipenv-python-version

@@ -22,7 +22,10 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
                     echo "$LATEST_2" > "$BUILD_DIR/runtime.txt"
                 fi
                 if [ "$PYTHON" = 3.6 ]; then
-                    echo "$LATEST_3" > "$BUILD_DIR/runtime.txt"
+                    echo "$LATEST_36" > "$BUILD_DIR/runtime.txt"
+                fi
+                if [ "$PYTHON" = 3.7 ]; then
+                    echo "$LATEST_37" > "$BUILD_DIR/runtime.txt"
                 fi
             fi
 

bin/steps/python

@@ -7,24 +7,62 @@ PYTHON_VERSION=$(cat runtime.txt)
 # The location of the pre-compiled python binary.
 VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"
 
-if [[ $PYTHON_VERSION =~ ^python-2 ]]; then
-  if [[ "$PYTHON_VERSION" != "$LATEST_2" ]]; then
-    puts-warn "The latest version of Python 2 is $LATEST_2 (you are using $PYTHON_VERSION, which is unsupported)."
-    puts-warn "We recommend upgrading by specifying the latest version ($LATEST_2)."
-    echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
+
+# check if runtime exists
+if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
+  if [[ "$PYTHON_VERSION" == $PY37* ]]; then
+    # do things to alert the user of security release available
+    if [ "$PYTHON_VERSION" != "$LATEST_37" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_37"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PY36* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_36" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_36"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PY35* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_35" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_35"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PY34* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_34" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_34"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PY27* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_27"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
   fi
 else
-  if [[ "$PYTHON_VERSION" != "$LATEST_3" ]]; then
-    puts-warn "The latest version of Python 3 is $LATEST_3 (you are using $PYTHON_VERSION, which is unsupported)."
-    puts-warn "We recommend upgrading by specifying the latest version ($LATEST_3)."
-    echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
-  fi
-
+  puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)."
+  puts-warn "Aborting.  More info: https://devcenter.heroku.com/articles/python-support"
+  exit 1
 fi
 
 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
     puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache"
-    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
+    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
+fi
+
+# need to clear the cache for first time installing SQLite3,
+# since the version is changing and could lead to runtime errors
+# with compiled extensions.
+if [ -d .heroku/python ] && [ ! -f .heroku/python-sqlite3-version ] && python_sqlite3_check "$PYTHON_VERSION"; then
+  puts-step "Need to update SQLite3, clearing cache"
+  rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
 fi
 
 if [ -f .heroku/python-version ]; then
@@ -36,6 +74,7 @@ if [ -f .heroku/python-version ]; then
   fi
 fi
 
+
 if [ ! "$SKIP_INSTALL" ]; then
     puts-step "Installing $PYTHON_VERSION"
 
@@ -58,6 +97,7 @@ if [ ! "$SKIP_INSTALL" ]; then
   hash -r
 fi
 
+
 # If Pip isn't up to date:
 if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
 

bin/steps/sqlite3

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
+
+sqlite3_version() {
+  SQLITE3_VERSION=${SQLITE3_VERSION:-$(dpkg -s libsqlite3-0 | grep Version | sed 's/Version: //')}
+
+  export SQLITE3_VERSION
+}
+
+sqlite3_install() {
+  HEROKU_PYTHON_DIR="$1"
+  SQLITE3_VERSION="$2"
+  HEADERS_ONLY="$3"
+
+  mkdir -p "$HEROKU_PYTHON_DIR"
+
+  APT_CACHE_DIR="$HEROKU_PYTHON_DIR/apt/cache"
+  APT_STATE_DIR="$HEROKU_PYTHON_DIR/apt/state"
+
+  mkdir -p "$APT_CACHE_DIR/archives/partial"
+  mkdir -p "$APT_STATE_DIR/lists/partial"
+
+  APT_OPTIONS="-o debug::nolocking=true"
+  APT_OPTIONS="$APT_OPTIONS -o dir::cache=$APT_CACHE_DIR"
+  APT_OPTIONS="$APT_OPTIONS -o dir::state=$APT_STATE_DIR"
+  APT_OPTIONS="$APT_OPTIONS -o dir::etc::sourcelist=/etc/apt/sources.list"
+
+  apt-get $APT_OPTIONS update > /dev/null 2>&1
+  if [ -z "$HEADERS_ONLY" ]; then
+    apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION" sqlite3="$SQLITE3_VERSION" > /dev/null 2>&1
+  else
+    apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION"
+  fi
+
+  find "$APT_CACHE_DIR/archives/" -name "*.deb" -exec dpkg -x {} "$HEROKU_PYTHON_DIR/sqlite3/" \;
+
+  mkdir -p "$HEROKU_PYTHON_DIR/include"
+  mkdir -p "$HEROKU_PYTHON_DIR/lib"
+
+  # remove old sqlite3 libraries/binaries
+  find "$HEROKU_PYTHON_DIR/include/" -name "sqlite3*.h" -exec rm -f {} \;
+  find "$HEROKU_PYTHON_DIR/lib/" -name "libsqlite3.*" -exec rm -f {} \;
+  rm -f "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc"
+  rm -f "$HEROKU_PYTHON_DIR/bin/sqlite3"
+
+  # copy over sqlite3 headers & bins and setup linking against the stack image library
+  mv "$HEROKU_PYTHON_DIR/sqlite3/usr/include/"* "$HEROKU_PYTHON_DIR/include/"
+  mv "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu"/libsqlite3.*a "$HEROKU_PYTHON_DIR/lib/"
+  mkdir -p "$HEROKU_PYTHON_DIR/lib/pkgconfig"
+  # set the right prefix/lib directories
+  sed -e 's/prefix=\/usr/prefix=\/app\/.heroku\/python/' -e 's/\/x86_64-linux-gnu//' "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/pkgconfig/sqlite3.pc" > "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc"
+  # need to point the libsqlite3.so to the stack image library for /usr/bin/ld -lsqlite3
+  SQLITE3_LIBFILE="/usr/lib/x86_64-linux-gnu/$(readlink -n "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/libsqlite3.so")"
+  ln -s "$SQLITE3_LIBFILE" "$HEROKU_PYTHON_DIR/lib/libsqlite3.so"
+  if [ -z "$HEADERS_ONLY" ]; then
+    mv "$HEROKU_PYTHON_DIR/sqlite3/usr/bin"/* "$HEROKU_PYTHON_DIR/bin/"
+  fi
+
+  # cleanup
+  rm -rf "$HEROKU_PYTHON_DIR/sqlite3/"
+  rm -rf "$HEROKU_PYTHON_DIR/apt/"
+}
+
+buildpack_sqlite3_install() {
+  sqlite3_version
+  HEROKU_PYTHON_DIR="$BUILD_DIR/.heroku/python"
+
+  SQLITE3_VERSION_FILE="$BUILD_DIR/.heroku/python-sqlite3-version"
+  if [ -f "$SQLITE3_VERSION_FILE" ]; then
+    INSTALLED_SQLITE3_VERSION=$(cat "$SQLITE3_VERSION_FILE")
+  fi
+
+  # python version check
+  if python_sqlite3_check "$PYTHON_VERSION"; then
+    # only install if the sqlite3 version has changed
+    if [ "$INSTALLED_SQLITE3_VERSION" != "$SQLITE3_VERSION" ]; then
+      puts-step "Installing SQLite3"
+      sqlite3_install "$BUILD_DIR/.heroku/python" "$SQLITE3_VERSION"
+
+      # save version installed
+      mkdir -p "$CACHE_DIR/.heroku/"
+      echo "$SQLITE3_VERSION" > "$CACHE_DIR/.heroku/python-sqlite3-version"
+    fi
+  fi
+}

bin/utils

@@ -58,3 +58,40 @@ measure-size() {
   echo "$(du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')"
 }
 
+# Python version operator >
+version_gt() {
+  test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1";
+}
+
+# Python verison operator >=
+version_gte() {
+  if [ "$1" == "$2" ]; then
+    return 0
+  fi
+
+  version_gt "$1" "$2"
+}
+
+# Check if Python 2
+python2_check() {
+  VERSION="$1"
+
+  version_gte "$VERSION" "python-2.7.0" && version_gt "python-3.0.0" "$VERSION"
+}
+
+# Check if Python 3
+python3_check() {
+  VERSION="$1"
+
+  version_gte "$VERSION" "python-3.0.0" && version_gt "python-4.0.0" "$VERSION"
+}
+
+# Check if Python version needs to install SQLite3
+python_sqlite3_check() {
+  VERSION="$1"
+  MIN_PYTHON_3="python-3.5.6"
+  MIN_PYTHON_2="python-2.7.15"
+
+  ( python2_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_2" ) \
+    || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" )
+}

builds/runtimes/python-2.7.15

@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
 
 OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
 
 echo "Building Python…"
 SOURCE_TARBALL='https://python.org/ftp/python/2.7.15/Python-2.7.15.tgz'

builds/runtimes/python-3.4.9

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.4.9/Python-3.4.9.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.4.9 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.5.6

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.5.6/Python-3.5.6.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.5.6 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.6.6

@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
 
 OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
 
 echo "Building Python…"
 SOURCE_TARBALL='https://python.org/ftp/python/3.6.6/Python-3.6.6.tgz'

builds/runtimes/python-3.6.7

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.6.7/Python-3.6.7.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.6.7 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.7.0

@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
 
 OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
 
 echo "Building Python…"
 SOURCE_TARBALL='https://python.org/ftp/python/3.7.0/Python-3.7.0.tgz'

builds/runtimes/python-3.7.1

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.7.1/Python-3.7.1.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.7.1 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

etc/ci-setup.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+sudo apt-get -qq update
+sudo apt-get install software-properties-common
+curl --fail --retry 3 --retry-delay 1 --connect-timeout 3 --max-time 30 https://cli-assets.heroku.com/install-ubuntu.sh | sh

etc/publish.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+
+BP_NAME=${1:-"heroku/python"}
+
+curVersion=$(heroku buildpacks:versions "$BP_NAME" | awk 'FNR == 3 { print $1 }')
+newVersion="v$((curVersion + 1))"
+
+read -p "Deploy as version: $newVersion [y/n]? " choice
+case "$choice" in
+  y|Y ) echo "";;
+  n|N ) exit 0;;
+  * ) exit 1;;
+esac
+
+originMaster=$(git rev-parse origin/master)
+echo "Tagging commit $originMaster with $newVersion... "
+git tag "$newVersion" "${originMaster:?}"
+git push origin refs/tags/$newVersion
+
+heroku buildpacks:publish "$BP_NAME" "$newVersion"
+
+if [ $(git tag | grep -q previous-version) ]; then
+    echo "Updating previous-version tag"
+    git tag -d previous-version
+    git push origin :previous-version
+    git tag previous-version latest-version
+fi
+if [ $(git tag | grep -q latest-version) ]; then
+    echo "Updating latest-version tag"
+    git tag -d latest-version
+    git push origin :latest-version
+    git tag latest-version "${originMaster:?}"
+    git push --tags
+fi
+
+echo "Done."

hatchet.json

@@ -0,0 +1,5 @@
+{
+  "python": [
+    "heroku/python-getting-started"
+  ]
+}

hatchet.lock

@@ -0,0 +1,3 @@
+---
+- - "./repos/python/python-getting-started"
+  - f56b90499ec11e1c9576da2f8c7331300e189db3

requirements.txt

@@ -1,3 +1,3 @@
 docopt==0.6.2
-bob-builder==0.0.13
+bob-builder
 boto==2.48.0

spec/hatchet/python_spec.rb

@@ -0,0 +1,10 @@
+require_relative '../spec_helper'
+
+describe "Python!!!!!!!!!!!" do
+  it "🐍" do
+    Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app|
+      expect(app.output).to           match(/Installing pip/)
+      expect(app.run('python -V')).to match(/3.6.6/)
+    end
+  end
+end

spec/spec_helper.rb

@@ -0,0 +1,23 @@
+ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/' + ENV['TRAVIS_REPO_SLUG'] + '.git'
+
+require 'rspec/core'
+require 'rspec/retry'
+require 'hatchet'
+
+require 'date'
+
+RSpec.configure do |config|
+  config.full_backtrace      = true
+  config.verbose_retry       = true # show retry status in spec process
+  config.default_retry_count = 2 if ENV['IS_RUNNING_ON_CI'] # retry all tests that fail again
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end
+
+if ENV['TRAVIS']
+  # Don't execute tests against "merge" commits
+  exit 0 if ENV['TRAVIS_PULL_REQUEST'] != 'false' && ENV['TRAVIS_BRANCH'] == 'master'
+end
+
+DEFAULT_STACK = 'heroku-16'

test/fixtures/airflow/requirements.txt

@@ -0,0 +1 @@
+apache-airflow==1.10

test/fixtures/nltk/runtime.txt

@@ -1 +1 @@
-python-2.7.13
+python-3.6.6

test/fixtures/python2_warn/runtime.txt

@@ -0,0 +1 @@
+python-2.7.14

test/fixtures/python3_4/requirements.txt

@@ -0,0 +1 @@
+flask

test/fixtures/python3_4/runtime.txt

@@ -0,0 +1 @@
+python-3.4.9

test/fixtures/python3_4_warn/requirements.txt

@@ -0,0 +1 @@
+flask

test/fixtures/python3_4_warn/runtime.txt

@@ -0,0 +1 @@
+python-3.4.0

test/fixtures/python3_5/requirements.txt

@@ -0,0 +1 @@
+requests

test/fixtures/python3_5/runtime.txt

@@ -0,0 +1 @@
+python-3.5.6

test/fixtures/python3_5_warn/requirements.txt

@@ -0,0 +1 @@
+flask

test/fixtures/python3_5_warn/runtime.txt

@@ -0,0 +1 @@
+python-3.5.3

test/fixtures/python3_6/requirements.txt

@@ -0,0 +1 @@
+requests

test/fixtures/python3_6/runtime.txt

@@ -0,0 +1 @@
+python-3.6.6

test/fixtures/python3_6_7/requirements.txt

@@ -0,0 +1 @@
+flask

test/fixtures/python3_6_7/runtime.txt

@@ -0,0 +1 @@
+python-3.6.7

test/fixtures/python3_6_warn/requirements.txt

@@ -0,0 +1 @@
+requests

test/fixtures/python3_6_warn/runtime.txt

@@ -0,0 +1 @@
+python-3.6.5

test/fixtures/python3_7/requirements.txt

@@ -0,0 +1 @@
+requests

test/fixtures/python3_7/runtime.txt

@@ -0,0 +1 @@
+python-3.7.0

test/fixtures/python3_7_1/requirements.txt

@@ -0,0 +1 @@
+flask

test/fixtures/python3_7_1/runtime.txt

@@ -0,0 +1 @@
+python-3.7.1

test/run

@@ -1,5 +1,12 @@
 #!/usr/bin/env bash
 
+testAirflow() {
+  export SLUGIFY_USES_TEXT_UNIDECODE="yes"
+  compile "airflow"
+  assertCaptured "apache-airflow==1.10"
+  assertCapturedSuccess
+}
+
 testPipenv() {
   compile "pipenv"
   assertCapturedSuccess
@@ -39,6 +46,11 @@ testGEOS() {
 }
 
 testNLTK() {
+  # NOTE: This is a RuntimeWarning emitted by Python 3's runpy.py script
+  # which is what is used when you call `python -m <module>`. This is due to
+  # how nltk imports things. It's not actually an error, but it would probably
+  # be bad to silence in Production.
+  export PYTHONWARNINGS="ignore::RuntimeWarning"
   compile "nltk"
   assertCaptured "Downloading NLTK packages: city_database stopwords"
   assertCapturedSuccess
@@ -75,18 +87,98 @@ testPylibmc() {
   assertCapturedSuccess
 }
 
+testPython2_warn() {
+    compile "python2_warn"
+    if [[ $STACK = "heroku-18" ]]; then
+      assertCapturedError
+    else
+      assertCaptured "python-2.7.14"
+      assertCaptured "security update!"
+      assertCapturedSuccess
+    fi
+}
+
 testPython2() {
-  compile "python2"
-  assertCaptured "python-2.7.15"
+    compile "python2"
+    assertCaptured "python-2.7.15"
+    assertCapturedSuccess
+}
+
+# This fail
+testPython3_4_warn() {
+  compile "python3_4_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCaptured "python-3.4.0"
+    assertCapturedSuccess
+  else
+    assertCapturedError
+  fi
+}
+
+# This fail
+testPython3_4() {
+  compile "python3_4"
+  assertCaptured "python-3.4.9"
+  assertCapturedError
+}
+
+# This fail
+testPython3_5_warn() {
+  compile "python3_5_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCaptured "python-3.5.3"
+    assertCaptured "security update!"
+    assertCapturedError
+  else
+    assertCapturedError
+  fi
+}
+
+# This will fail
+testPython3_5() {
+  compile "python3_5"
+  assertCaptured "python-3.5.6"
+  assertCapturedError
+}
+
+# This will warn
+testPython3_6_warn() {
+  compile "python3_6_warn"
+  assertCaptured "python-3.6.5"
+  assertCaptured "security update!"
   assertCapturedSuccess
 }
 
-testPython3() {
-  compile "python3"
+testPython3_6() {
+  compile "python3_6"
   assertCaptured "python-3.6.6"
   assertCapturedSuccess
 }
 
+# This will fail
+testPython3_6_7() {
+  compile "python3_6_7"
+  assertCaptured "python-3.6.7"
+  assertCapturedError
+}
+
+testPython3_7() {
+  compile "python3_7"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "python-3.7.0"
+    assertCapturedSuccess
+  fi
+}
+
+# This will fail
+testPython3_7_1() {
+  compile "python3_7_1"
+  assertCaptured "python-3.7.1"
+  assertCapturedError
+}
+
 testGitEgg() {
   compile "git-egg"
   assertCaptured "requests"

tests.sh

@@ -13,4 +13,9 @@ fi
 if [[ "$STACK" == "heroku-16" ]]; then
     make test-heroku-16
     exit $?
-fi
+fi
+
+if [[ "$STACK" == "heroku-18" ]]; then
+    make test-heroku-18
+    exit $?
+fi