mirror of
https://github.com/kennethreitz/heroku-buildpack-python.git
synced 2026-06-05 15:00:19 +00:00
Ed Morley
54115fc89b
So that it links to the changelog post now that it exists: https://devcenter.heroku.com/changelog-items/1947 And to adjust the emphasis towards the migration guide over the Geo buildpack repository link. Closes @W-8391415@.
379 lines
15 KiB
Bash
Executable File
379 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
|
||
# The Heroku Python Buildpack. This script accepts parameters for a build
|
||
# directory, a cache directory, and a directory for app environment variables.
|
||
|
||
# Warning: there are a few hacks in this script to accommodate excellent builds
|
||
# on Heroku. No guarantee for external compatibility is made. However,
|
||
# everything should work fine outside of the Heroku environment, if the
|
||
# environment is setup correctly.
|
||
|
||
# Usage:
|
||
#
|
||
# $ bin/compile <build-dir> <cache-dir> <env-path>
|
||
|
||
# Fail fast and fail hard.
|
||
set -eo pipefail
|
||
|
||
# Used by buildpack-stdlib's metrics features.
|
||
export BPLOG_PREFIX="buildpack.python"
|
||
export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null}
|
||
|
||
[ "$BUILDPACK_XTRACE" ] && set -o xtrace
|
||
|
||
# Prepend proper path for old-school virtualenv hackery.
|
||
# This may not be neccessary.
|
||
export PATH=:/usr/local/bin:$PATH
|
||
|
||
# Setup Path variables, for later use in the Buildpack.
|
||
BIN_DIR=$(cd "$(dirname "$0")"; pwd) # absolute path
|
||
ROOT_DIR=$(dirname "$BIN_DIR")
|
||
BUILD_DIR=$1
|
||
CACHE_DIR=$2
|
||
ENV_DIR=$3
|
||
|
||
# Export Path variables, for use in sub-scripts.
|
||
export BUILD_DIR CACHE_DIR ENV_DIR
|
||
|
||
# Set the base URL for downloading buildpack assets like Python runtimes.
|
||
# The user can provide BUILDPACK_S3_BASE_URL to specify a custom target.
|
||
# Note: this is designed for non-Heroku use, as it does not use the user-provided
|
||
# environment variable mechanism (the ENV_DIR).
|
||
S3_BASE_URL="${BUILDPACK_S3_BASE_URL:-"https://heroku-buildpack-python.s3.amazonaws.com"}"
|
||
# This has to be exported since it's used by the geo-libs step which is run in a subshell.
|
||
|
||
# Default Python Versions
|
||
# shellcheck source=bin/default_pythons
|
||
source "$BIN_DIR/default_pythons"
|
||
|
||
# Supported Python Branches
|
||
PY39="python-3.9"
|
||
PY38="python-3.8"
|
||
PY37="python-3.7"
|
||
PY36="python-3.6"
|
||
PY35="python-3.5"
|
||
PY34="python-3.4"
|
||
PY27="python-2.7"
|
||
PYPY27="pypy2.7"
|
||
PYPY36="pypy3.6"
|
||
|
||
# Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
|
||
# TODO: Remove this and require that STACK be set explicitly.
|
||
DEFAULT_PYTHON_STACK="heroku-18"
|
||
|
||
# Common Problem Warnings:
|
||
# This section creates a temporary file in which to stick the output of `pip install`.
|
||
# The `warnings` subscript then greps through this for common problems and guides
|
||
# the user towards resolution of known issues.
|
||
WARNINGS_LOG=$(mktemp)
|
||
RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
|
||
|
||
# The buildpack ships with a few executable tools.
|
||
# This installs them into the path, so we can execute them directly.
|
||
export PATH=$PATH:$ROOT_DIR/vendor/
|
||
|
||
# Set environment variables if they weren't set by the platform.
|
||
[ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK
|
||
|
||
# Sanitize externally-provided environment variables:
|
||
# The following environment variables are either problematic or simply unneccessary
|
||
# for the buildpack to have knowledge of, so we unset them, to keep the environment
|
||
# as clean and pristine as possible.
|
||
unset PYTHONHOME PYTHONPATH
|
||
|
||
# Import the utils script, which contains helper functions used throughout the buildpack.
|
||
# shellcheck source=bin/utils
|
||
source "$BIN_DIR/utils"
|
||
|
||
# Import the warnings script, which contains the `pip install` user warning mechanisms
|
||
# (mentioned and explained above)
|
||
# shellcheck source=bin/warnings
|
||
source "$BIN_DIR/warnings"
|
||
|
||
if [[ "${STACK}" == "cedar-14" ]]; then
|
||
mcount "failure.unsupported.cedar-14"
|
||
puts-warn "The Cedar-14 stack is no longer supported by the latest release of this buildpack."
|
||
puts-warn
|
||
puts-warn "Please switch to the Cedar-14 support branch by using this buildpack URL:"
|
||
puts-warn "https://github.com/heroku/heroku-buildpack-python#cedar-14"
|
||
puts-warn
|
||
puts-warn "For instructions on how to change the buildpacks used by an app, see:"
|
||
puts-warn "https://devcenter.heroku.com/articles/buildpacks#setting-a-buildpack-on-an-application"
|
||
exit 1
|
||
fi
|
||
|
||
if [[ -f "${ENV_DIR}/BUILD_WITH_GEO_LIBRARIES" ]]; then
|
||
mcount "failure.unsupported.BUILD_WITH_GEO_LIBRARIES"
|
||
puts-warn "The Python buildpack's legacy BUILD_WITH_GEO_LIBRARIES functonality is"
|
||
puts-warn "no longer supported:"
|
||
puts-warn "https://devcenter.heroku.com/changelog-items/1947"
|
||
puts-warn
|
||
puts-warn "To continue to use GDAL, GEOS or PROJ support, see the migration guide:"
|
||
puts-warn "https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-no-longer-supported"
|
||
puts-warn
|
||
puts-warn "Or if you no longer need those libraries, this message can be hidden by"
|
||
puts-warn "unsetting the BUILD_WITH_GEO_LIBRARIES environment variable, using:"
|
||
puts-warn "heroku config:unset BUILD_WITH_GEO_LIBRARIES"
|
||
exit 1
|
||
fi
|
||
|
||
# Make the directory in which we will create symlinks from the temporary build directory
|
||
# to `/app`.
|
||
# Symlinks are required, since Python is not a portable installation.
|
||
# More on this topic later.
|
||
mkdir -p /app/.heroku
|
||
|
||
# This buildpack programatically generates (or simply copies) a number of files for
|
||
# buildpack machinery: an export script, and a number of `.profile.d` scripts. This
|
||
# section declares the locations of those files and targets.
|
||
PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
|
||
EXPORT_PATH="$BIN_DIR/../export"
|
||
GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh"
|
||
WEB_CONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/WEB_CONCURRENCY.sh"
|
||
|
||
# We'll need to send these statics to other scripts we `source`.
|
||
export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH
|
||
|
||
# Python Environment Variables
|
||
# Set Python-specific environment variables, for running Python within the buildpack.
|
||
# Notes on each variable included.
|
||
|
||
# PATH is relatively obvious, we need to be able to execute 'python'.
|
||
export PATH=/app/.heroku/python/bin:/app/.heroku/vendor/bin:$PATH
|
||
# Tell Python to not buffer it's stdin/stdout.
|
||
export PYTHONUNBUFFERED=1
|
||
# Set the locale to a well-known and expected standard.
|
||
export LANG=en_US.UTF-8
|
||
# `~/.heroku/vendor` is an place where the buildpack may stick pre-build binaries for known
|
||
# C dependencies. This section configures Python (GCC, more specifically)
|
||
# and pip to automatically include these paths when building binaries.
|
||
# TODO: Stop adding .heroku/vendor here now that the buildpack no longer vendors anything.
|
||
export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH
|
||
export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH
|
||
export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH
|
||
export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH
|
||
export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH
|
||
|
||
# Global pip options (https://pip.pypa.io/en/stable/user_guide/#environment-variables).
|
||
# Disable pip's warnings about EOL Python since we show our own.
|
||
export PIP_NO_PYTHON_VERSION_WARNING=1
|
||
|
||
# The Application Code
|
||
# --------------------
|
||
|
||
# Switch to the repo's context.
|
||
cd "$BUILD_DIR"
|
||
|
||
# The Cache
|
||
# ---------
|
||
|
||
# The workflow for the Python Buildpack's cache is as follows:
|
||
#
|
||
# - `~/.heroku/{known-paths}` are copied from the cache into the slug.
|
||
# - The build is executed, modifying `~/.heroku/{known-paths}`.
|
||
# - Once the build is complete, `~/.heroku/{known-paths}` is copied back into the cache.
|
||
|
||
# Create the cache directory, if it doesn't exist.
|
||
mkdir -p "$CACHE_DIR/.heroku"
|
||
|
||
# Restore old artifacts from the cache.
|
||
mkdir -p .heroku
|
||
|
||
# The Python installation.
|
||
cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true
|
||
# A plain text file which contains the current stack being used (used for cache busting).
|
||
cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
|
||
# A plain text file which contains the current python version being used (used for cache busting).
|
||
cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
|
||
# A plain text file which contains the current sqlite3 version being used (used for cache busting).
|
||
cp -R "$CACHE_DIR/.heroku/python-sqlite3-version" .heroku/ &> /dev/null || true
|
||
# "editable" installations of code repositories, via pip or pipenv.
|
||
if [[ -d "$CACHE_DIR/.heroku/src" ]]; then
|
||
cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true
|
||
fi
|
||
|
||
# The pre_compile hook. Customers rely on this. Don't remove it.
|
||
# This part of the code is used to allow users to customize their build experience
|
||
# without forking the buildpack by providing a `bin/pre_compile` script, which gets
|
||
# run inline with the buildpack automatically.
|
||
|
||
# shellcheck source=bin/steps/hooks/pre_compile
|
||
source "$BIN_DIR/steps/hooks/pre_compile"
|
||
|
||
# Sticky runtimes. If there was a previous build, and it used a given version of Python,
|
||
# continue to use that version of Python in perpituity (warnings will be raised if
|
||
# they are out–of–date).
|
||
if [ -f "$CACHE_DIR/.heroku/python-version" ]; then
|
||
CACHED_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version")
|
||
fi
|
||
|
||
# We didn't always record the stack version. This code is in place because of that.
|
||
if [ -f "$CACHE_DIR/.heroku/python-stack" ]; then
|
||
CACHED_PYTHON_STACK=$(cat "$CACHE_DIR/.heroku/python-stack")
|
||
else
|
||
CACHED_PYTHON_STACK=$STACK
|
||
fi
|
||
|
||
# Pipenv Python version support.
|
||
# Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version).
|
||
# Convert it to a runtime.txt file.
|
||
|
||
# shellcheck source=bin/steps/pipenv-python-version
|
||
source "$BIN_DIR/steps/pipenv-python-version"
|
||
|
||
if [[ -f runtime.txt ]]; then
|
||
mcount "version.reason.python.specified"
|
||
elif [[ -n "${CACHED_PYTHON_VERSION:-}" ]]; then
|
||
mcount "version.reason.python.cached"
|
||
echo "${CACHED_PYTHON_VERSION}" > runtime.txt
|
||
else
|
||
mcount "version.reason.python.default"
|
||
echo "${DEFAULT_PYTHON_VERSION}" > runtime.txt
|
||
fi
|
||
|
||
# Create the directory for .profile.d, if it doesn't exist.
|
||
mkdir -p "$(dirname "$PROFILE_PATH")"
|
||
# Create the directory for editable source code installation, if it doesn't exist.
|
||
mkdir -p /app/.heroku/src
|
||
|
||
# On Heroku CI, builds happen in `/app`. Otherwise, on the Heroku platform,
|
||
# they occur in a temp directory. Beacuse Python is not portable, we must create
|
||
# symlinks to emulate that we are operating in `/app` during the build process.
|
||
# This is (hopefully obviously) because apps end up running from `/app` in production.
|
||
if [[ $BUILD_DIR != '/app' ]]; then
|
||
# python expects to reside in /app, so set up symlinks
|
||
# we will not remove these later so subsequent buildpacks can still invoke it
|
||
ln -nsf "$BUILD_DIR/.heroku/python" /app/.heroku/python
|
||
ln -nsf "$BUILD_DIR/.heroku/vendor" /app/.heroku/vendor
|
||
# Note: .heroku/src is copied in later.
|
||
fi
|
||
|
||
# Download / Install Python, from pre-build binaries available on Amazon S3.
|
||
# This step also bootstraps pip / setuptools.
|
||
(( start=$(nowms) ))
|
||
# shellcheck source=bin/steps/python
|
||
source "$BIN_DIR/steps/python"
|
||
mtime "python.install.time" "${start}"
|
||
|
||
# Install Pipenv dependencies, if a Pipfile was provided.
|
||
# shellcheck source=bin/steps/pipenv
|
||
source "$BIN_DIR/steps/pipenv"
|
||
|
||
# If no requirements.txt file given, assume `setup.py develop` is intended.
|
||
# This allows for people to ship a setup.py application to Heroku
|
||
|
||
if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
|
||
echo "-e ." > requirements.txt
|
||
fi
|
||
|
||
# Fix egg-links.
|
||
# Because we're installing things into a different path than we're running them (temp dir vs app dir),
|
||
# We must re-write all of Python's eggpath links to target the proper directory.
|
||
# shellcheck source=bin/steps/eggpath-fix
|
||
source "$BIN_DIR/steps/eggpath-fix"
|
||
|
||
# SQLite3 support.
|
||
# This sets up and installs sqlite3 dev headers and the sqlite3 binary but not the
|
||
# libsqlite3-0 library since that exists on the stack image.
|
||
# Note: This only applies to Python 2.7.15+ and Python 3.6.6+
|
||
(( start=$(nowms) ))
|
||
# shellcheck source=bin/steps/sqlite3
|
||
source "$BIN_DIR/steps/sqlite3"
|
||
buildpack_sqlite3_install
|
||
mtime "sqlite3.install.time" "${start}"
|
||
|
||
# pip install
|
||
# -----------
|
||
|
||
# Install dependencies with pip (where the magic happens).
|
||
(( start=$(nowms) ))
|
||
# shellcheck source=bin/steps/pip-install
|
||
source "$BIN_DIR/steps/pip-install"
|
||
mtime "pip.install.time" "${start}"
|
||
|
||
# Support for NLTK corpora.
|
||
# Note: this may only work on Python 2.7. I don't think many customers use this functionality,
|
||
# and it should probably be undocumented.
|
||
# (there's an import error on 3.6 that should hopefully be fixed upstream at some point)
|
||
(( start=$(nowms) ))
|
||
sub_env "$BIN_DIR/steps/nltk"
|
||
mtime "nltk.download.time" "${start}"
|
||
|
||
# Support for editable installations. Here, we are copying pip–created src directory,
|
||
# and copying it into the proper place (the logical place to do this was early, but it must be done here).
|
||
# In CI, $BUILD_DIR is /app.
|
||
if [[ ! "$BUILD_DIR" == "/app" ]]; then
|
||
rm -rf "$BUILD_DIR/.heroku/src"
|
||
deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src"
|
||
fi
|
||
|
||
|
||
# Django collectstatic support.
|
||
# The buildpack automatically runs collectstatic for Django applications.
|
||
# This is the cause for the majority of build failures on the Python platform.
|
||
# These failures are intentional — if collectstatic (which can be tricky, at times) fails,
|
||
# your build fails.
|
||
(( start=$(nowms) ))
|
||
sub_env "$BIN_DIR/steps/collectstatic"
|
||
mtime "collectstatic.time" "${start}"
|
||
|
||
|
||
# Progamatically create .profile.d script for application runtime environment variables.
|
||
|
||
# Set the PATH to include Python / pip / pipenv / etc.
|
||
set_env PATH "\$HOME/.heroku/python/bin:\$PATH"
|
||
# Tell Python to run in unbuffered mode.
|
||
set_env PYTHONUNBUFFERED true
|
||
# Tell Python where it lives.
|
||
set_env PYTHONHOME "\$HOME/.heroku/python"
|
||
# Set variables for C libraries.
|
||
set_env LIBRARY_PATH "\$HOME/.heroku/vendor/lib:\$HOME/.heroku/python/lib:\$LIBRARY_PATH"
|
||
set_env LD_LIBRARY_PATH "\$HOME/.heroku/vendor/lib:\$HOME/.heroku/python/lib:\$LD_LIBRARY_PATH"
|
||
# Locale.
|
||
set_default_env LANG en_US.UTF-8
|
||
# The Python hash seed is set to random.
|
||
set_default_env PYTHONHASHSEED random
|
||
# Tell Python to look for Python modules in the /app dir. Don't change this.
|
||
set_default_env PYTHONPATH "\$HOME"
|
||
|
||
# Python expects to be in /app, if at runtime, it is not, set
|
||
# up symlinks… this can occur when the subdir buildpack is used.
|
||
cat <<EOT >> "$PROFILE_PATH"
|
||
if [[ \$HOME != "/app" ]]; then
|
||
mkdir -p /app/.heroku
|
||
ln -nsf "\$HOME/.heroku/python" /app/.heroku/python
|
||
ln -nsf "\$HOME/.heroku/vendor" /app/.heroku/vendor
|
||
fi
|
||
EOT
|
||
|
||
# Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.
|
||
cp "$ROOT_DIR/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH"
|
||
cp "$ROOT_DIR/vendor/python.gunicorn.sh" "$GUNICORN_PROFILE_PATH"
|
||
|
||
# Experimental post_compile hook. Don't remove this.
|
||
# shellcheck source=bin/steps/hooks/post_compile
|
||
source "$BIN_DIR/steps/hooks/post_compile"
|
||
|
||
# Fix egg-links, again.
|
||
# shellcheck source=bin/steps/eggpath-fix2
|
||
source "$BIN_DIR/steps/eggpath-fix2"
|
||
|
||
# Store new artifacts in the cache.
|
||
rm -rf "$CACHE_DIR/.heroku/python"
|
||
rm -rf "$CACHE_DIR/.heroku/python-version"
|
||
rm -rf "$CACHE_DIR/.heroku/python-stack"
|
||
rm -rf "$CACHE_DIR/.heroku/vendor"
|
||
rm -rf "$CACHE_DIR/.heroku/src"
|
||
|
||
mkdir -p "$CACHE_DIR/.heroku"
|
||
cp -R .heroku/python "$CACHE_DIR/.heroku/"
|
||
cp -R .heroku/python-version "$CACHE_DIR/.heroku/"
|
||
cp -R .heroku/python-stack "$CACHE_DIR/.heroku/" &> /dev/null || true
|
||
if [[ -d .heroku/src ]]; then
|
||
cp -R .heroku/src "$CACHE_DIR/.heroku/" &> /dev/null || true
|
||
fi
|
||
|
||
# Measure the size of the Python installation.
|
||
# shellcheck disable=SC2119
|
||
mmeasure 'python.size' "$(measure-size)"
|