kennethreitz/heroku-buildpack-python
1
0
Fork 0
mirror of https://github.com/kennethreitz/heroku-buildpack-python.git synced 2026-06-05 23:10:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #799 from heroku/build-cleanup

Browse Source 
Build cleanup
This commit is contained in:
Casey
2019-09-11 13:34:56 -07:00
committed by GitHub
parent 05b7ceb82c 006ea77641
commit b8d1035231
23 changed files with 227 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3
View File
@@ -7,3 +7,6 @@ repos/*
#Venv
buildpack/*
builds/dockerenv.staging*
builds/dockerenv.production
.travis.yml
+6 -4
View File
@@ -9,10 +9,12 @@ before_script:
- bundle exec hatchet ci:setup
script:
- docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/Dockerfile .
- docker run --rm -e "STACK=heroku-16" heroku-python-build-heroku-16 bash $TESTFOLDER
- docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/Dockerfile.heroku-18 .
- docker run --rm -e "STACK=heroku-18" heroku-python-build-heroku-18 bash $TESTFOLDER
- docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
- docker run --rm -e "STACK=cedar-14" travis-build-cedar-14 bash $TESTFOLDER
- docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
- docker run --rm -e "STACK=heroku-16" travis-build-heroku-16 bash $TESTFOLDER
- docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
- docker run --rm -e "STACK=heroku-18" travis-build-heroku-18 bash $TESTFOLDER
jobs:
include:
Makefile
+7 -2
View File
@@ -8,6 +8,11 @@ check:
@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
@shellcheck -x bin/steps/hooks/*
test-cedar-14:
@echo "Running tests in docker (cedar-14)..."
@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar-14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
@echo ""
test-heroku-16:
@echo "Running tests in docker (heroku-16)..."
@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
@@ -21,7 +26,7 @@ test-heroku-18:
buildenv-heroku-16:
@echo "Creating build environment (heroku-16)..."
@echo
@docker build --pull -t python-buildenv-heroku-16 .
@docker build --pull -f $(shell pwd)/builds/heroku-16.Dockerfile -t python-buildenv-heroku-16 .
@echo
@echo "Usage..."
@echo
@@ -34,7 +39,7 @@ buildenv-heroku-16:
buildenv-heroku-18:
@echo "Creating build environment (heroku-18)..."
@echo
@docker build --pull -f Dockerfile.heroku-18 -t python-buildenv-heroku-18 .
@docker build --pull -f $(shell pwd)/builds/heroku-18.Dockerfile -t python-buildenv-heroku-18 .
@echo
@echo "Usage..."
@echo
builds/README.md
+41 -31
View File
@@ -1,47 +1,57 @@
# Python Buildpack Binaries
For Cedar-14 stack
------------------
## Building the Docker Images
To get started with it, create an app on Heroku inside a clone of this repository, and set your S3 config vars:
**After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack:
$ heroku create --buildpack https://github.com/heroku/heroku-buildpack-python#not-heroku
$ heroku config:set WORKSPACE_DIR=builds
$ heroku config:set AWS_ACCESS_KEY_ID=<your_aws_key>
$ heroku config:set AWS_SECRET_ACCESS_KEY=<your_aws_secret>
$ heroku config:set S3_BUCKET=<your_s3_bucket_name>
$ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
$ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
$ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
## Using the Image
Then, shell into an instance and run a build by giving the name of the formula inside `builds`:
You can e.g. `bash` into each of the images you built using their tag:
$ heroku run bash
Running `bash` attached to terminal... up, run.6880
~ $ bob build runtimes/python-2.7.6
docker run --rm -ti heroku-python-build-cedar-14 bash
docker run --rm -ti heroku-python-build-heroku-16 bash
docker run --rm -ti heroku-python-build-heroku-18 bash
Fetching dependencies... found 2:
- libraries/sqlite
You then have a shell where you can run `bob build`, `bob deploy`, and so forth. You can of course also invoke these programs directly with `docker run`:
Building formula runtimes/python-2.7.6:
=== Building Python 2.7.6
Fetching Python v2.7.6 source...
Compiling...
docker run --rm -ti heroku-python-build-heroku-18 bob build runtimes/python-2.7.15
If this works, run `bob deploy` instead of `bob build` to have the result uploaded to S3 for you.
In order to `bob deploy`, AWS credentials must be set up, as well as name and prefix of your custom S3 bucket (unless you're deploying to the Heroku production buckets that are pre-defined in each `Dockerfile`); see next section for details.
To speed things up drastically, it'll usually be a good idea to `heroku run bash --size PX` instead.
## Configuration
For Heroku-16 stack
-------------------
File `dockerenv.default` contains a list of required env vars; most of these have default values defined in `Dockerfile`. You can copy this file to a location outside the buildpack and modify it with the values you desire and pass its location with `--env-file`, or pass the env vars to `docker run` using `--env`.
1. Ensure GNU Make and Docker are installed.
2. From the root of the buildpack repository, run: `make buildenv-heroku-16`
3. Follow the instructions displayed!
Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCKET`, `S3_PREFIX`, and `S3_REGION`. If you're building your own packages, you'll likely want to change `S3_BUCKET` and `S3_PREFIX` to match your info. Instead of setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` into that file, you may also pass them to `docker run` through the environment, or explicitly using `--env`, in order to prevent accidental commits of credentials.
For Heroku-18 stack
-------------------
### Passing AWS credentials to the container
1. Ensure GNU Make and Docker are installed.
2. From the root of the buildpack repository, run: `make buildenv-heroku-18`
3. Follow the instructions displayed!
If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file.
Enjoy :)
#### Passing credentials explicitly
docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash
#### Passing credentials through the environment
The two environment variables `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY` are defined in `builds/dockerenv.default`, without values. This will cause Docker to "forward" values for these variables from the current environment, so you can pass them in:
AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
or
export AWS_ACCESS_KEY_ID=...
export AWS_SECRET_ACCESS_KEY=...
docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
#### Passing credentials through a separate env file
This method is the easiest for users who want to build packages in their own S3 bucket, as they will have to adjust the `S3_BUCKET` and `S3_PREFIX` environment variable values anyway from their default values.
For this method, it is important to keep the credentials file in a location outside the buildpack, so that your credentials aren't accidentally committed. Copy `builds/dockerenv.default` **to a safe location outside the buildpack directory**, and insert your values for `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY`.
docker run --rm -ti --env-file=../SOMEPATHOUTSIDE/s3.env heroku-python-build-heroku-18 bash
builds/cedar-14.Dockerfile
+15
View File
@@ -0,0 +1,15 @@
FROM heroku/cedar:14
WORKDIR /app
ENV WORKSPACE_DIR="/app/builds" \
S3_BUCKET="lang-python" \
S3_PREFIX="cedar-14/" \
DEBIAN_FRONTEND=noninteractive \
STACK="cedar-14"
RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
COPY requirements.txt /app/
RUN pip install -r /app/requirements.txt
COPY . /app
builds/dockerenv.default
+5
View File
@@ -0,0 +1,5 @@
AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY
S3_BUCKET
S3_PREFIX
S3_REGION
Dockerfile → builds/heroku-16.Dockerfile
+2 -1
View File
@@ -4,7 +4,8 @@ WORKDIR /app
ENV WORKSPACE_DIR="/app/builds" \
S3_BUCKET="lang-python" \
S3_PREFIX="heroku-16/" \
DEBIAN_FRONTEND=noninteractive
DEBIAN_FRONTEND=noninteractive \
STACK="heroku-16"
RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
Dockerfile.heroku-18 → builds/heroku-18.Dockerfile
+2 -1
View File
@@ -4,7 +4,8 @@ WORKDIR /app
ENV WORKSPACE_DIR="/app/builds" \
S3_BUCKET="lang-python" \
S3_PREFIX="heroku-18/" \
DEBIAN_FRONTEND=noninteractive
DEBIAN_FRONTEND=noninteractive \
STACK="heroku-18"
RUN apt-get update && apt-get install --no-install-recommends -y python-pip-whl=9.0.1-2 python-pip=9.0.1-2 python-setuptools python-wheel && rm -rf /var/lib/apt/lists/*
builds/runtimes/pypy
Executable
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
# fail hard
set -o pipefail
# fail harder
set -eu
OUT_PREFIX=$1
dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
dep_name=$(basename $BASH_SOURCE) # this is us
dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name
dep_package=${dep_name}2-v${dep_version} # it's always "pypy2-…"
dep_dirname=${dep_package}-linux64
dep_archive_name=${dep_dirname}.tar.bz2
dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
echo "Building PyPy…"
curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python"
builds/runtimes/pypy-5.3.1
+1 -10
View File
@@ -2,13 +2,4 @@
# Build Path: /app/.heroku/python/
# Build Deps: libraries/sqlite
# NOTICE: This formula only works for the cedar-14 stack, not cedar.
OUT_PREFIX=$1
echo "Building PyPy…"
SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux64.tar.bz2'
curl -L $SOURCE_TARBALL | tar jx
cp -R pypy2-v5.3.1-linux64/* $OUT_PREFIX
ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python
source $(dirname $0)/pypy
builds/runtimes/pypy-5.6.0
+1 -10
View File
@@ -2,13 +2,4 @@
# Build Path: /app/.heroku/python/
# Build Deps: libraries/sqlite
# NOTICE: This formula only works for the cedar-14 stack, not cedar.
OUT_PREFIX=$1
echo "Building PyPy…"
SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux64.tar.bz2'
curl -L $SOURCE_TARBALL | tar jx
cp -R pypy2-v5.6.0-linux64/* $OUT_PREFIX
ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python
source $(dirname $0)/pypy
builds/runtimes/pypy-5.7.0
+1 -10
View File
@@ -2,13 +2,4 @@
# Build Path: /app/.heroku/python/
# Build Deps: libraries/sqlite
# NOTICE: This formula only works for the cedar-14 and heroku-16 stacks, not cedar.
OUT_PREFIX=$1
echo "Building PyPy…"
SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-linux64.tar.bz2'
curl -L $SOURCE_TARBALL | tar jx
cp -R pypy2-v5.7.0-linux64/* $OUT_PREFIX
ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python
source $(dirname $0)/pypy
builds/runtimes/pypy-5.7.1
+1 -10
View File
@@ -2,13 +2,4 @@
# Build Path: /app/.heroku/python/
# Build Deps: libraries/sqlite
# NOTICE: This formula only works for the cedar-14 stack, not cedar.
OUT_PREFIX=$1
echo "Building PyPy…"
SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux64.tar.bz2'
curl -L $SOURCE_TARBALL | tar jx
cp -R pypy2-v5.7.1-linux64/* $OUT_PREFIX
ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python
source $(dirname $0)/pypy
builds/runtimes/pypy-5.8.0
+1 -8
View File
@@ -2,11 +2,4 @@
# Build Path: /app/.heroku/python/
# Build Deps: libraries/sqlite
OUT_PREFIX=$1
echo "Building PyPy…"
SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux64.tar.bz2'
curl -L $SOURCE_TARBALL | tar jx
cp -R pypy2-v5.8.0-linux64/* $OUT_PREFIX
ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python
source $(dirname $0)/pypy
builds/runtimes/pypy3
Executable
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
# fail hard
set -o pipefail
# fail harder
set -eu
OUT_PREFIX=$1
dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
dep_name=$(basename $BASH_SOURCE) # this is us
dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name
dep_package=${dep_name}${dep_version_prefix:-}-v${dep_version}${dep_version_suffix:-}
dep_dirname=${dep_package}-linux64
dep_archive_name=${dep_dirname}.tar.bz2
dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
echo "Building PyPy3…"
curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
ln "$OUT_PREFIX/bin/pypy3" "$OUT_PREFIX/bin/python"
builds/runtimes/pypy3-2.4.0
+1 -10
View File
@@ -2,13 +2,4 @@
# Build Path: /app/.heroku/python/
# Build Deps: libraries/sqlite
# NOTICE: This formula only works for the cedar-14 stack, not cedar.
OUT_PREFIX=$1
echo "Building PyPy…"
SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-2.4.0-linux64.tar.bz2'
curl -L $SOURCE_TARBALL | tar jx
cp -R pypy3-2.4.0-linux64/* $OUT_PREFIX
ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python
source $(dirname $0)/pypy3
builds/runtimes/pypy3-5.5.0
+4 -10
View File
@@ -2,13 +2,7 @@
# Build Path: /app/.heroku/python/
# Build Deps: libraries/sqlite
# NOTICE: This formula only works for the cedar-14 stack, not cedar.
OUT_PREFIX=$1
echo "Building PyPy…"
SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux64.tar.bz2'
curl -L $SOURCE_TARBALL | tar jx
cp -R pypy3-v5.5.0-linux64/* $OUT_PREFIX
ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python
# version pypy3-5.5.0 is really called pypy3.3-v5.5.0-alpha
dep_version_prefix=".3"
dep_version_suffix="-alpha"
source $(dirname $0)/pypy3
builds/runtimes/pypy3-5.7.1
+1 -8
View File
@@ -2,11 +2,4 @@
# Build Path: /app/.heroku/python/
# Build Deps: libraries/sqlite
OUT_PREFIX=$1
echo "Building PyPy…"
SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-linux64.tar.bz2'
curl -L $SOURCE_TARBALL | tar jx
cp -R pypy3-v5.7.1-linux64/* $OUT_PREFIX
ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python
source $(dirname $0)/pypy3
builds/runtimes/pypy3-5.8.0
+1 -8
View File
@@ -2,11 +2,4 @@
# Build Path: /app/.heroku/python/
# Build Deps: libraries/sqlite
OUT_PREFIX=$1
echo "Building PyPy…"
SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.8.0-linux64.tar.bz2'
curl -L $SOURCE_TARBALL | tar jx
cp -R pypy3-v5.8.0-linux64/* $OUT_PREFIX
ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python
source $(dirname $0)/pypy3
builds/runtimes/python-3.6.9
Executable
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
# Build Path: /app/.heroku/python/
# Build Deps: libraries/sqlite
source $(dirname $0)/python3
builds/runtimes/python-3.7.4
Executable
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
# Build Path: /app/.heroku/python/
# Build Deps: libraries/sqlite
source $(dirname $0)/python3
builds/runtimes/python2
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Build Path: /app/.heroku/python/
OUT_PREFIX=$1
BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
export BIN_DIR
# Orient ourselves and build download link
dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
BASE=${dep_formula##*/} # this gives us relative path
python_version=${BASE^} # this gives us only the filename with version number
version_number=$(echo "$python_version" | cut -d- -f2) # this returns just X.X.X
dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz
# shellcheck source=bin/utils
source "$BIN_DIR/steps/sqlite3"
sqlite3_version
echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
echo "Building Python 3..."
echo "Pulling from source: ${dep_url}"
curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}"
mv "${OUT_PREFIX}/${python_version}" src
cd src
./configure --prefix=$OUT_PREFIX --enable-unicode=ucs4 --with-ensurepip=no
make
make install
# Remove unneeded test directories, similar to the official Docker Python images:
# https://github.com/docker-library/python
find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
builds/runtimes/python3
Executable
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
# fail hard
set -o pipefail
# fail harder
set -eu
OUT_PREFIX=$1
BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
export BIN_DIR
# Orient ourselves and build download link
dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
BASE=${dep_formula##*/} # this gives us relative path
python_version=${BASE^} # this gives us only the filename with version number
version_number=$(echo "$python_version" | cut -d- -f2) # this returns just X.X.X
dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz
# shellcheck source=bin/utils
source "$BIN_DIR/steps/sqlite3"
sqlite3_version
echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
echo "Building Python 3..."
echo "Pulling from source: ${dep_url}"
curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}"
mv "${OUT_PREFIX}/${python_version}" src
cd src
./configure --prefix=$OUT_PREFIX --with-ensurepip=no
make
make install
# Remove unneeded test directories, similar to the official Docker Python images:
# https://github.com/docker-library/python
find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
# Remove spare /
LOCATION=${OUT_PREFIX%?}
ln $LOCATION/bin/python3 $LOCATION/bin/python