Release v184 (#1106)

Now `make deploy-runtimes` will build binaries for Heroku-20 by default
too, without the need to pass it in via an explicit `STACKS=...`list.

Closes @W-8233698@.

[skip changelog]

CHANGELOG.md

@@ -3,6 +3,11 @@
 ## Unreleased
 
 
+## v184 (2020-10-21)
+
+- Vendor buildpack-stdlib instead of fetching from S3 (#1100).
+- Fix metric names for metrics emitted within `sub_env` (#1099).
+
 ## v183 (2020-10-12)
 
 - Add support for Heroku-20 (#968).

Gemfile.lock

@@ -3,13 +3,13 @@ GEM
   specs:
     diff-lcs (1.4.4)
     erubis (2.7.0)
-    excon (0.76.0)
+    excon (0.78.0)
     heroics (0.1.1)
       erubis (~> 2.0)
       excon
       moneta
       multi_json (>= 1.9.2)
-    heroku_hatchet (7.3.1)
+    heroku_hatchet (7.3.3)
       excon (~> 0)
       platform-api (~> 3)
       rrrretry (~> 1)

Makefile

@@ -2,7 +2,7 @@
 .PHONY: check test builder-image buildenv deploy-runtimes tools
 
 STACK ?= heroku-18
-STACKS ?= cedar-14 heroku-16 heroku-18
+STACKS ?= cedar-14 heroku-16 heroku-18 heroku-20
 TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
 ENV_FILE ?= builds/dockerenv.default
 BUILDER_IMAGE_PREFIX := heroku-python-build

bin/compile

@@ -15,10 +15,8 @@
 # Fail fast and fail hard.
 set -eo pipefail
 
-# Boostrap the Buildpack Standard Library.
-# Disable unused env var warning since shellcheck doesn't know about the stdlib.
-# shellcheck disable=2034
-BPLOG_PREFIX="buildpack.python"
+# Used by buildpack-stdlib's metrics features.
+export BPLOG_PREFIX="buildpack.python"
 export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
 
 [ "$BUILDPACK_XTRACE" ] && set -o xtrace

bin/utils

@@ -2,12 +2,13 @@
 shopt -s extglob
 shopt -s nullglob
 
-# The standard library.
-if [[ ! -f  /tmp/stdlib.sh ]]; then
-  curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v8/stdlib.sh > /tmp/stdlib.sh
-fi
-# shellcheck source=/dev/null
-source /tmp/stdlib.sh
+# This is necessary since this script is sometimes sourced from
+# subshells that don't have the variables from bin/compile.
+# Remove this once we no longer wrap all the things in `sub_env`.
+BIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT_DIR=$(dirname "${BIN_DIR}")
+# shellcheck source=vendor/buildpack-stdlib_v8.sh
+source "${ROOT_DIR}/vendor/buildpack-stdlib_v8.sh"
 
 if [ "$(uname)" == Darwin ]; then
     sed() { command sed -l "$@"; }

test/run-features

@@ -116,6 +116,7 @@ testHooks() {
   local expected_env_vars=(
     _
     BIN_DIR
+    BPLOG_PREFIX
     BUILD_DIR
     BUILDPACK_LOG_FILE
     CACHE_DIR

vendor/buildpack-stdlib_v8.sh

@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+
+# From:
+# https://raw.githubusercontent.com/heroku/buildpack-stdlib/v8/stdlib.sh
+
+# Buildpack defaults
+# ---------------
+
+export BUILDPACK_LOG_FILE="${BUILDPACK_LOG_FILE:-/dev/null}"
+
+# Standard Output
+# ---------------
+
+# Buildpack Steps.
+puts_step() {
+  if [[ "$*" == "-" ]]; then
+    read -r output
+  else
+    output=$*
+  fi
+  echo -e "\\e[1m\\e[36m=== $output\\e[0m"
+  unset output
+}
+
+# Buildpack Error.
+puts_error() {
+  if [[ "$*" == "-" ]]; then
+    read -r output
+  else
+    output=$*
+  fi
+  echo -e "\\e[1m\\e[31m=!= $output\\e[0m"
+}
+
+# Buildpack Warning.
+puts_warn() {
+  if [[ "$*" == "-" ]]; then
+    read -r output
+  else
+    output=$*
+  fi
+  echo -e "\\e[1m\\e[33m=!= $output\\e[0m"
+}
+
+# Is verbose set?
+is_verbose() {
+  if [[ -n $BUILDPACK_VERBOSE ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+# Buildpack Verbose.
+puts_verbose() {
+  if is_verbose; then
+    if [[ "$*" == "-" ]]; then
+      read -r output
+    else
+      output=$*
+    fi
+    echo "$output"
+    unset output
+  fi
+}
+
+# Buildpack Utilities
+# -------------------
+
+# Usage: $ set-env key value
+# NOTICE: Expects PROFILE_PATH & EXPORT_PATH to be set!
+set_env() {
+  # TODO: automatically create profile path directory if it doesn't exist.
+  echo "export $1=$2" >> "$PROFILE_PATH"
+  echo "export $1=$2" >> "$EXPORT_PATH"
+}
+
+# Usage: $ set-default-env key value
+# NOTICE: Expects PROFILE_PATH & EXPORT_PATH to be set!
+set_default_env() {
+  echo "export $1=\${$1:-$2}" >> "$PROFILE_PATH"
+  echo "export $1=\${$1:-$2}" >> "$EXPORT_PATH"
+}
+
+# Usage: $ un-set-env key
+# NOTICE: Expects PROFILE_PATH to be set!
+un_set_env() {
+  echo "unset $1" >> "$PROFILE_PATH"
+}
+
+# Usage: $ _env-blacklist pattern
+# Outputs a regex of default blacklist env vars.
+_env_blacklist() {
+  local regex=${1:-''}
+  if [ -n "$regex" ]; then
+    regex="|$regex"
+  fi
+  echo "^(PATH|GIT_DIR|CPATH|CPPATH|LD_PRELOAD|LIBRARY_PATH$regex)$"
+}
+
+# Usage: $ export-env ENV_DIR WHITELIST BLACKLIST
+# Exports the environment variables defined in the given directory.
+export_env() {
+  local env_dir=${1:-$ENV_DIR}
+  local whitelist=${2:-''}
+  local blacklist
+  blacklist="$(_env_blacklist "$3")"
+  if [ -d "$env_dir" ]; then
+    # Environment variable names won't contain characters affected by:
+    # shellcheck disable=SC2045
+    for e in $(ls "$env_dir"); do
+      echo "$e" | grep -E "$whitelist" | grep -qvE "$blacklist" &&
+      export "$e=$(cat "$env_dir/$e")"
+      :
+    done
+  fi
+}
+
+# Usage: $ sub-env command
+# Runs a subshell of specified command with user-provided config.
+# NOTICE: Expects ENV_DIR to be set. WHITELIST & BLACKLIST are optional.
+# Examples:
+#    WHITELIST=${2:-''}
+#    BLACKLIST=${3:-'^(GIT_DIR|PYTHONHOME|LD_LIBRARY_PATH|LIBRARY_PATH|PATH)$'}
+sub_env() {
+  (
+    # TODO: Fix https://github.com/heroku/buildpack-stdlib/issues/37
+    # shellcheck disable=SC2153
+    export_env "$ENV_DIR" "$WHITELIST" "$BLACKLIST"
+
+    "$@"
+  )
+}
+
+# Logging
+# -------
+
+# Notice: These functions expect BPLOG_PREFIX and BUILDPACK_LOG_FILE to be defined (BUILDPACK_LOG_FILE can point to /dev/null if not provided by the buildpack).
+# Example: BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}; BPLOG_PREFIX="buildpack.go"
+
+# Returns now, in milleseconds. Useful for logging.
+# Example: $ let start=$(nowms); sleep 30; mtime "glide.install.time" "${start}"
+nowms() {
+  date +%s%3N
+}
+
+# Log arbitrary data to the logfile (e.g. a packaging file).
+# Usage: $ bplog "$(<${vendorJSON})
+bplog() {
+  echo -n "${@}" | awk 'BEGIN {printf "msg=\""; f="%s"} {gsub(/"/, "\\\"", $0); printf f, $0} {if (NR == 1) f="\\n%s" } END { print "\"" }' >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Measures time elapsed for a specific build step.
+# Usage: $ let start=$(nowms); mtime "glide.install.time" "${start}"
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#distributions-measure
+mtime() {
+  local key="${BPLOG_PREFIX}.${1}"
+  local start="${2}"
+  local end="${3:-$(nowms)}"
+  echo "${key} ${start} ${end}" | awk '{ printf "measure#%s=%.3f\n", $1, ($3 - $2)/1000 }' >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Logs a count for a specific built step.
+# Usage: $ mcount "tool.govendor"
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#counting-count
+mcount() {
+  local k="${BPLOG_PREFIX}.${1}"
+  local v="${2:-1}"
+  echo "count#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Logs a measure for a specific build step.
+# Usage: $ mmeasure "tool.installed_dependencies" 42
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#distributions-measure
+mmeasure() {
+  local k="${BPLOG_PREFIX}.${1}"
+  local v="${2}"
+  echo "measure#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Logs a unuique measurement build step.
+# Usage: $ munique "versions.count" 2.7.13
+# https://github.com/heroku/engineering-docs/blob/master/guides/logs-as-data.md#uniques-unique
+munique() {
+  local k="${BPLOG_PREFIX}.${1}"
+  local v="${2}"
+  echo "unique#${k}=${v}" >> "${BUILDPACK_LOG_FILE}"
+}
+
+# Measures when an exit path to the buildpack is reached, given a name, then exits 1.
+# Usage: $ mcount-exi "binExists"
+mcount_exit() {
+  mcount "error.${1}"
+  exit 1
+}