From e28c1012b7199566511849bd21deb66cd532ea3f Mon Sep 17 00:00:00 2001 From: David Zuelke Date: Wed, 13 Feb 2019 18:34:58 +0100 Subject: [PATCH 01/23] reuse logic in runtimes/pypy formulae --- builds/runtimes/pypy | 22 ++++++++++++++++++++++ builds/runtimes/pypy-5.3.1 | 11 +---------- builds/runtimes/pypy-5.6.0 | 11 +---------- builds/runtimes/pypy-5.7.0 | 11 +---------- builds/runtimes/pypy-5.7.1 | 11 +---------- builds/runtimes/pypy-5.8.0 | 9 +-------- 6 files changed, 27 insertions(+), 48 deletions(-) create mode 100755 builds/runtimes/pypy diff --git a/builds/runtimes/pypy b/builds/runtimes/pypy new file mode 100755 index 0000000..d2142a1 --- /dev/null +++ b/builds/runtimes/pypy @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# fail hard +set -o pipefail +# fail harder +set -eu + +OUT_PREFIX=$1 + +dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 +dep_name=$(basename $BASH_SOURCE) # this is us +dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name +dep_package=${dep_name}2-v${dep_version} # it's always "pypy2-…" +dep_dirname=${dep_package}-linux64 +dep_archive_name=${dep_dirname}.tar.bz2 +dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name} + +echo "Building PyPy…" + +curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name + +ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python" diff --git a/builds/runtimes/pypy-5.3.1 b/builds/runtimes/pypy-5.3.1 index cc1be00..97ff496 100755 --- a/builds/runtimes/pypy-5.3.1 +++ b/builds/runtimes/pypy-5.3.1 @@ -2,13 +2,4 @@ # Build Path: /app/.heroku/python/ # Build Deps: libraries/sqlite -# NOTICE: This formula only works for the cedar-14 stack, not cedar. - -OUT_PREFIX=$1 - -echo "Building PyPy…" -SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux64.tar.bz2' -curl -L $SOURCE_TARBALL | tar jx -cp -R pypy2-v5.3.1-linux64/* $OUT_PREFIX - -ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python +source $(dirname $0)/pypy diff --git a/builds/runtimes/pypy-5.6.0 b/builds/runtimes/pypy-5.6.0 index 685fa1d..97ff496 100755 --- a/builds/runtimes/pypy-5.6.0 +++ b/builds/runtimes/pypy-5.6.0 @@ -2,13 +2,4 @@ # Build Path: /app/.heroku/python/ # Build Deps: libraries/sqlite -# NOTICE: This formula only works for the cedar-14 stack, not cedar. - -OUT_PREFIX=$1 - -echo "Building PyPy…" -SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux64.tar.bz2' -curl -L $SOURCE_TARBALL | tar jx -cp -R pypy2-v5.6.0-linux64/* $OUT_PREFIX - -ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python +source $(dirname $0)/pypy diff --git a/builds/runtimes/pypy-5.7.0 b/builds/runtimes/pypy-5.7.0 index c16d5e3..97ff496 100755 --- a/builds/runtimes/pypy-5.7.0 +++ b/builds/runtimes/pypy-5.7.0 @@ -2,13 +2,4 @@ # Build Path: /app/.heroku/python/ # Build Deps: libraries/sqlite -# NOTICE: This formula only works for the cedar-14 and heroku-16 stacks, not cedar. - -OUT_PREFIX=$1 - -echo "Building PyPy…" -SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-linux64.tar.bz2' -curl -L $SOURCE_TARBALL | tar jx -cp -R pypy2-v5.7.0-linux64/* $OUT_PREFIX - -ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python +source $(dirname $0)/pypy diff --git a/builds/runtimes/pypy-5.7.1 b/builds/runtimes/pypy-5.7.1 index 86e34c6..97ff496 100755 --- a/builds/runtimes/pypy-5.7.1 +++ b/builds/runtimes/pypy-5.7.1 @@ -2,13 +2,4 @@ # Build Path: /app/.heroku/python/ # Build Deps: libraries/sqlite -# NOTICE: This formula only works for the cedar-14 stack, not cedar. - -OUT_PREFIX=$1 - -echo "Building PyPy…" -SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux64.tar.bz2' -curl -L $SOURCE_TARBALL | tar jx -cp -R pypy2-v5.7.1-linux64/* $OUT_PREFIX - -ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python +source $(dirname $0)/pypy diff --git a/builds/runtimes/pypy-5.8.0 b/builds/runtimes/pypy-5.8.0 index 6e8b6ac..97ff496 100755 --- a/builds/runtimes/pypy-5.8.0 +++ b/builds/runtimes/pypy-5.8.0 @@ -2,11 +2,4 @@ # Build Path: /app/.heroku/python/ # Build Deps: libraries/sqlite -OUT_PREFIX=$1 - -echo "Building PyPy…" -SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux64.tar.bz2' -curl -L $SOURCE_TARBALL | tar jx -cp -R pypy2-v5.8.0-linux64/* $OUT_PREFIX - -ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python +source $(dirname $0)/pypy From b52fac4d32d2de079432efbc125c928678dd1bab Mon Sep 17 00:00:00 2001 From: David Zuelke Date: Wed, 13 Feb 2019 18:50:18 +0100 Subject: [PATCH 02/23] reuse logic in runtimes/pypy3 formulae --- builds/runtimes/pypy3 | 22 ++++++++++++++++++++++ builds/runtimes/pypy3-2.4.0 | 11 +---------- builds/runtimes/pypy3-5.5.0 | 14 ++++---------- builds/runtimes/pypy3-5.7.1 | 9 +-------- builds/runtimes/pypy3-5.8.0 | 9 +-------- 5 files changed, 29 insertions(+), 36 deletions(-) create mode 100755 builds/runtimes/pypy3 diff --git a/builds/runtimes/pypy3 b/builds/runtimes/pypy3 new file mode 100755 index 0000000..6d11006 --- /dev/null +++ b/builds/runtimes/pypy3 @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# fail hard +set -o pipefail +# fail harder +set -eu + +OUT_PREFIX=$1 + +dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 +dep_name=$(basename $BASH_SOURCE) # this is us +dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name +dep_package=${dep_name}${dep_version_prefix:-}-v${dep_version}${dep_version_suffix:-} +dep_dirname=${dep_package}-linux64 +dep_archive_name=${dep_dirname}.tar.bz2 +dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name} + +echo "Building PyPy3…" + +curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name + +ln "$OUT_PREFIX/bin/pypy3" "$OUT_PREFIX/bin/python" diff --git a/builds/runtimes/pypy3-2.4.0 b/builds/runtimes/pypy3-2.4.0 index 61b1638..7deb446 100755 --- a/builds/runtimes/pypy3-2.4.0 +++ b/builds/runtimes/pypy3-2.4.0 @@ -2,13 +2,4 @@ # Build Path: /app/.heroku/python/ # Build Deps: libraries/sqlite -# NOTICE: This formula only works for the cedar-14 stack, not cedar. - -OUT_PREFIX=$1 - -echo "Building PyPy…" -SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-2.4.0-linux64.tar.bz2' -curl -L $SOURCE_TARBALL | tar jx -cp -R pypy3-2.4.0-linux64/* $OUT_PREFIX - -ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python +source $(dirname $0)/pypy3 diff --git a/builds/runtimes/pypy3-5.5.0 b/builds/runtimes/pypy3-5.5.0 index fde9ffe..f934d00 100755 --- a/builds/runtimes/pypy3-5.5.0 +++ b/builds/runtimes/pypy3-5.5.0 @@ -2,13 +2,7 @@ # Build Path: /app/.heroku/python/ # Build Deps: libraries/sqlite -# NOTICE: This formula only works for the cedar-14 stack, not cedar. - -OUT_PREFIX=$1 - -echo "Building PyPy…" -SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux64.tar.bz2' -curl -L $SOURCE_TARBALL | tar jx -cp -R pypy3-v5.5.0-linux64/* $OUT_PREFIX - -ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python +# version pypy3-5.5.0 is really called pypy3.3-v5.5.0-alpha +dep_version_prefix=".3" +dep_version_suffix="-alpha" +source $(dirname $0)/pypy3 diff --git a/builds/runtimes/pypy3-5.7.1 b/builds/runtimes/pypy3-5.7.1 index fb0e718..7deb446 100755 --- a/builds/runtimes/pypy3-5.7.1 +++ b/builds/runtimes/pypy3-5.7.1 @@ -2,11 +2,4 @@ # Build Path: /app/.heroku/python/ # Build Deps: libraries/sqlite -OUT_PREFIX=$1 - -echo "Building PyPy…" -SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-linux64.tar.bz2' -curl -L $SOURCE_TARBALL | tar jx -cp -R pypy3-v5.7.1-linux64/* $OUT_PREFIX - -ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python +source $(dirname $0)/pypy3 diff --git a/builds/runtimes/pypy3-5.8.0 b/builds/runtimes/pypy3-5.8.0 index df08b9f..7deb446 100755 --- a/builds/runtimes/pypy3-5.8.0 +++ b/builds/runtimes/pypy3-5.8.0 @@ -2,11 +2,4 @@ # Build Path: /app/.heroku/python/ # Build Deps: libraries/sqlite -OUT_PREFIX=$1 - -echo "Building PyPy…" -SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.8.0-linux64.tar.bz2' -curl -L $SOURCE_TARBALL | tar jx -cp -R pypy3-v5.8.0-linux64/* $OUT_PREFIX - -ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python +source $(dirname $0)/pypy3 From a771b7bf948f609bfb7cec2eb4838f18f9358ce6 Mon Sep 17 00:00:00 2001 From: David Zuelke Date: Wed, 13 Feb 2019 22:51:32 +0100 Subject: [PATCH 03/23] move Dockerfiles to central location, add all stacks, update README --- builds/README.md | 72 +++++++++++-------- builds/cedar-14.Dockerfile | 13 ++++ builds/dockerenv.default | 5 ++ Dockerfile => builds/heroku-16.Dockerfile | 0 .../heroku-18.Dockerfile | 0 5 files changed, 59 insertions(+), 31 deletions(-) create mode 100644 builds/cedar-14.Dockerfile create mode 100644 builds/dockerenv.default rename Dockerfile => builds/heroku-16.Dockerfile (100%) rename Dockerfile.heroku-18 => builds/heroku-18.Dockerfile (100%) diff --git a/builds/README.md b/builds/README.md index 1b4b77f..200893a 100644 --- a/builds/README.md +++ b/builds/README.md @@ -1,47 +1,57 @@ # Python Buildpack Binaries -For Cedar-14 stack ------------------- +## Building the Docker Images -To get started with it, create an app on Heroku inside a clone of this repository, and set your S3 config vars: +**After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack: - $ heroku create --buildpack https://github.com/heroku/heroku-buildpack-python#not-heroku - $ heroku config:set WORKSPACE_DIR=builds - $ heroku config:set AWS_ACCESS_KEY_ID= - $ heroku config:set AWS_SECRET_ACCESS_KEY= - $ heroku config:set S3_BUCKET= + $ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile . + $ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile . + $ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile . +## Using the Image -Then, shell into an instance and run a build by giving the name of the formula inside `builds`: +You can e.g. `bash` into each of the images you built using their tag: - $ heroku run bash - Running `bash` attached to terminal... up, run.6880 - ~ $ bob build runtimes/python-2.7.6 + docker run --rm -ti heroku-python-build-cedar-14 bash + docker run --rm -ti heroku-python-build-heroku-16 bash + docker run --rm -ti heroku-python-build-heroku-18 bash - Fetching dependencies... found 2: - - libraries/sqlite +You then have a shell where you can run `bob build`, `bob deploy`, and so forth. You can of course also invoke these programs directly with `docker run`: - Building formula runtimes/python-2.7.6: - === Building Python 2.7.6 - Fetching Python v2.7.6 source... - Compiling... + docker run --rm -ti heroku-python-build-heroku-18 bob build runtimes/python-2.7.15 -If this works, run `bob deploy` instead of `bob build` to have the result uploaded to S3 for you. +In order to `bob deploy`, AWS credentials must be set up, as well as name and prefix of your custom S3 bucket (unless you're deploying to the Heroku production buckets that are pre-defined in each `Dockerfile`); see next section for details. -To speed things up drastically, it'll usually be a good idea to `heroku run bash --size PX` instead. +## Configuration -For Heroku-16 stack -------------------- +File `dockerenv.default` contains a list of required env vars; most of these have default values defined in `Dockerfile`. You can copy this file to a location outside the buildpack and modify it with the values you desire and pass its location with `--env-file`, or pass the env vars to `docker run` using `--env`. -1. Ensure GNU Make and Docker are installed. -2. From the root of the buildpack repository, run: `make buildenv-heroku-16` -3. Follow the instructions displayed! +Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCKET`, `S3_PREFIX`, and `S3_REGION`. If you're building your own packages, you'll likely want to change `S3_BUCKET` and `S3_PREFIX` to match your info. Instead of setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` into that file, you may also pass them to `docker run` through the environment, or explicitly using `--env`, in order to prevent accidental commits of credentials. -For Heroku-18 stack -------------------- +### Passing AWS credentials to the container -1. Ensure GNU Make and Docker are installed. -2. From the root of the buildpack repository, run: `make buildenv-heroku-18` -3. Follow the instructions displayed! +If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file. -Enjoy :) +#### Passing credentials explicitly + + docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash + +#### Passing credentials through the environment + +The two environment variables `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY` are defined in `builds/dockerenv.default`, without values. This will cause Docker to "forward" values for these variables from the current environment, so you can pass them in: + + AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash + +or + + export AWS_ACCESS_KEY_ID=... + export AWS_SECRET_ACCESS_KEY=... + docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash + +#### Passing credentials through a separate env file + +This method is the easiest for users who want to build packages in their own S3 bucket, as they will have to adjust the `S3_BUCKET` and `S3_PREFIX` environment variable values anyway from their default values. + +For this method, it is important to keep the credentials file in a location outside the buildpack, so that your credentials aren't accidentally committed. Copy `builds/dockerenv.default` **to a safe location outside the buildpack directory**, and insert your values for `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY`. + + docker run --rm -ti --env-file=../SOMEPATHOUTSIDE/s3.env heroku-python-build-heroku-18 bash diff --git a/builds/cedar-14.Dockerfile b/builds/cedar-14.Dockerfile new file mode 100644 index 0000000..921b437 --- /dev/null +++ b/builds/cedar-14.Dockerfile @@ -0,0 +1,13 @@ +FROM heroku/cedar:14 + +WORKDIR /app +ENV WORKSPACE_DIR="/app/builds" \ + S3_BUCKET="lang-python" \ + S3_PREFIX="cedar-14/" + +RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/ +RUN pip install -r /app/requirements.txt + +COPY . /app diff --git a/builds/dockerenv.default b/builds/dockerenv.default new file mode 100644 index 0000000..9d2b8c2 --- /dev/null +++ b/builds/dockerenv.default @@ -0,0 +1,5 @@ +AWS_ACCESS_KEY_ID +AWS_SECRET_ACCESS_KEY +S3_BUCKET +S3_PREFIX +S3_REGION diff --git a/Dockerfile b/builds/heroku-16.Dockerfile similarity index 100% rename from Dockerfile rename to builds/heroku-16.Dockerfile diff --git a/Dockerfile.heroku-18 b/builds/heroku-18.Dockerfile similarity index 100% rename from Dockerfile.heroku-18 rename to builds/heroku-18.Dockerfile From 4f8bb77b13d9483db9830a14893c2e39289a8845 Mon Sep 17 00:00:00 2001 From: David Zuelke Date: Fri, 15 Feb 2019 03:42:33 +0100 Subject: [PATCH 04/23] expose $STACK env to each image so it can be used in formulae --- builds/cedar-14.Dockerfile | 4 +++- builds/heroku-16.Dockerfile | 3 ++- builds/heroku-18.Dockerfile | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/builds/cedar-14.Dockerfile b/builds/cedar-14.Dockerfile index 921b437..ea11542 100644 --- a/builds/cedar-14.Dockerfile +++ b/builds/cedar-14.Dockerfile @@ -3,7 +3,9 @@ FROM heroku/cedar:14 WORKDIR /app ENV WORKSPACE_DIR="/app/builds" \ S3_BUCKET="lang-python" \ - S3_PREFIX="cedar-14/" + S3_PREFIX="cedar-14/" \ + DEBIAN_FRONTEND=noninteractive \ + STACK="cedar-14" RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* diff --git a/builds/heroku-16.Dockerfile b/builds/heroku-16.Dockerfile index 0e00b3c..324042e 100644 --- a/builds/heroku-16.Dockerfile +++ b/builds/heroku-16.Dockerfile @@ -4,7 +4,8 @@ WORKDIR /app ENV WORKSPACE_DIR="/app/builds" \ S3_BUCKET="lang-python" \ S3_PREFIX="heroku-16/" \ - DEBIAN_FRONTEND=noninteractive + DEBIAN_FRONTEND=noninteractive \ + STACK="heroku-16" RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* diff --git a/builds/heroku-18.Dockerfile b/builds/heroku-18.Dockerfile index b4f8239..6428fa6 100644 --- a/builds/heroku-18.Dockerfile +++ b/builds/heroku-18.Dockerfile @@ -4,7 +4,8 @@ WORKDIR /app ENV WORKSPACE_DIR="/app/builds" \ S3_BUCKET="lang-python" \ S3_PREFIX="heroku-18/" \ - DEBIAN_FRONTEND=noninteractive + DEBIAN_FRONTEND=noninteractive \ + STACK="heroku-18" RUN apt-get update && apt-get install --no-install-recommends -y python-pip-whl=9.0.1-2 python-pip=9.0.1-2 python-setuptools python-wheel && rm -rf /var/lib/apt/lists/* From 97e3366409dfa3ff19e8cb694727592869861c69 Mon Sep 17 00:00:00 2001 From: David Zuelke Date: Wed, 13 Feb 2019 22:51:32 +0100 Subject: [PATCH 05/23] move Dockerfiles to central location, add all stacks, update README --- builds/README.md | 2 ++ ...edar-14.Dockerfile => cedar-14.Dockerfile~HEAD} | 0 ...central location, add all stacks, update README | 13 +++++++++++++ ...oku-18.Dockerfile => heroku-18.Dockerfile~HEAD} | 0 ...central location, add all stacks, update README | 14 ++++++++++++++ 5 files changed, 29 insertions(+) rename builds/{cedar-14.Dockerfile => cedar-14.Dockerfile~HEAD} (100%) create mode 100644 builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README rename builds/{heroku-18.Dockerfile => heroku-18.Dockerfile~HEAD} (100%) create mode 100644 builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README diff --git a/builds/README.md b/builds/README.md index 200893a..e07fdf8 100644 --- a/builds/README.md +++ b/builds/README.md @@ -32,6 +32,8 @@ Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCK If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file. +If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file. + #### Passing credentials explicitly docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash diff --git a/builds/cedar-14.Dockerfile b/builds/cedar-14.Dockerfile~HEAD similarity index 100% rename from builds/cedar-14.Dockerfile rename to builds/cedar-14.Dockerfile~HEAD diff --git a/builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README b/builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README new file mode 100644 index 0000000..921b437 --- /dev/null +++ b/builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README @@ -0,0 +1,13 @@ +FROM heroku/cedar:14 + +WORKDIR /app +ENV WORKSPACE_DIR="/app/builds" \ + S3_BUCKET="lang-python" \ + S3_PREFIX="cedar-14/" + +RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/ +RUN pip install -r /app/requirements.txt + +COPY . /app diff --git a/builds/heroku-18.Dockerfile b/builds/heroku-18.Dockerfile~HEAD similarity index 100% rename from builds/heroku-18.Dockerfile rename to builds/heroku-18.Dockerfile~HEAD diff --git a/builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README b/builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README new file mode 100644 index 0000000..466506a --- /dev/null +++ b/builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README @@ -0,0 +1,14 @@ +FROM heroku/heroku:18-build + +WORKDIR /app +ENV WORKSPACE_DIR="/app/builds" \ + S3_BUCKET="lang-python" \ + S3_PREFIX="heroku-18/" \ + DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/ +RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt + +COPY . /app From 8f1b24f64e52d8ff0006b4cbfab385d2d7d1da06 Mon Sep 17 00:00:00 2001 From: David Zuelke Date: Fri, 15 Feb 2019 03:42:33 +0100 Subject: [PATCH 06/23] expose $STACK env to each image so it can be used in formulae --- ...kerfiles to central location, add all stacks, update README | 3 ++- ...kerfiles to central location, add all stacks, update README | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README b/builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README index 921b437..f82fa16 100644 --- a/builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README +++ b/builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README @@ -3,7 +3,8 @@ FROM heroku/cedar:14 WORKDIR /app ENV WORKSPACE_DIR="/app/builds" \ S3_BUCKET="lang-python" \ - S3_PREFIX="cedar-14/" + S3_PREFIX="cedar-14/" \ + STACK="cedar-14" RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* diff --git a/builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README b/builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README index 466506a..6288ac0 100644 --- a/builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README +++ b/builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README @@ -4,7 +4,8 @@ WORKDIR /app ENV WORKSPACE_DIR="/app/builds" \ S3_BUCKET="lang-python" \ S3_PREFIX="heroku-18/" \ - DEBIAN_FRONTEND=noninteractive + DEBIAN_FRONTEND=noninteractive \ + STACK="heroku-18" RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* From e1d0fcae0e8fbb2cd463506cceeadea6e4905110 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Wed, 28 Aug 2019 13:55:07 -0700 Subject: [PATCH 07/23] remove Cedar from binary docs --- builds/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/builds/README.md b/builds/README.md index e07fdf8..7b39fde 100644 --- a/builds/README.md +++ b/builds/README.md @@ -4,7 +4,6 @@ **After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack: - $ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile . $ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile . $ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile . From 6bc4d42774f61c45f3367ebb166bfe3a8634c948 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Wed, 28 Aug 2019 14:02:30 -0700 Subject: [PATCH 08/23] first try for python-3.6.9 --- builds/runtimes/python-3.6.9 | 5 ++++ builds/runtimes/python3 | 56 ++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 builds/runtimes/python-3.6.9 create mode 100644 builds/runtimes/python3 diff --git a/builds/runtimes/python-3.6.9 b/builds/runtimes/python-3.6.9 new file mode 100644 index 0000000..dea889d --- /dev/null +++ b/builds/runtimes/python-3.6.9 @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ +# Build Deps: libraries/sqlite + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python3 b/builds/runtimes/python3 new file mode 100644 index 0000000..3a5e321 --- /dev/null +++ b/builds/runtimes/python3 @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# fail hard +set -o pipefail +# fail harder +set -eu + +OUT_PREFIX=$1 + +dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 +dep_name=$(basename $BASH_SOURCE) # this is us +dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name +dep_package=${dep_name}-${dep_version} # it's always "Python-…" +dep_dirname=${dep_package}-linux64 +dep_archive_name=${dep_dirname}.tar.bz2 +dep_url=https://python.org/ftp/python/${dep_version}/${dep_package}.tgz + + +echo "Installing necesary sqlite headers" +# install the necessary sqlite headers +apt-get update +apt-get install -y libsqlite3-dev + +# # Old way of installing sqlite +# # shellcheck source=bin/utils +# source "$BIN_DIR/steps/sqlite3" + +# sqlite3_version +# echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +# sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 + +echo "Building Python 3..." + +curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name + +mv "${dep_package}" src +cd src + +./configure --prefix=$OUT_PREFIX --with-ensurepip=no +make +make install + +# Remove unneeded test directories, similar to the official Docker Python images: +# https://github.com/docker-library/python +find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' + + +# Remove spare / +LOCATION=${OUT_PREFIX%?} + +ln $LOCATION/bin/python3 $LOCATION/bin/python + +# copy over sqlite3 headers +cp "/usr/include/sqlite3"* "$LOCATION/include/" + +# create generic .so symlink against stack image +ln -s "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0" "$LOCATION/lib/libsqlite3.so" From f3af6aa4a2f936b15f258ee3e6d6c94c00586dfd Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Wed, 28 Aug 2019 15:11:34 -0700 Subject: [PATCH 09/23] add python3 formula --- builds/runtimes/python3 | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/builds/runtimes/python3 b/builds/runtimes/python3 index 3a5e321..730e874 100644 --- a/builds/runtimes/python3 +++ b/builds/runtimes/python3 @@ -10,24 +10,24 @@ OUT_PREFIX=$1 dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 dep_name=$(basename $BASH_SOURCE) # this is us dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name -dep_package=${dep_name}-${dep_version} # it's always "Python-…" -dep_dirname=${dep_package}-linux64 -dep_archive_name=${dep_dirname}.tar.bz2 +dep_package=$(echo $dep_name | tr p P)-${dep_version} # it's always "Python-…" +dep_dirname=${dep_package}-linux64 # this is uppercased +dep_archive_name=${dep_dirname}.tar.bz2 # this is uppercased too dep_url=https://python.org/ftp/python/${dep_version}/${dep_package}.tgz -echo "Installing necesary sqlite headers" -# install the necessary sqlite headers -apt-get update -apt-get install -y libsqlite3-dev +# echo "Installing necesary sqlite headers" +# # install the necessary sqlite headers +# apt-get update +# apt-get install -y libsqlite3-dev -# # Old way of installing sqlite -# # shellcheck source=bin/utils -# source "$BIN_DIR/steps/sqlite3" +# Old way of installing sqlite +# shellcheck source=bin/utils +source "$BIN_DIR/steps/sqlite3" -# sqlite3_version -# echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" -# sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 +sqlite3_version +echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 echo "Building Python 3..." @@ -49,8 +49,8 @@ LOCATION=${OUT_PREFIX%?} ln $LOCATION/bin/python3 $LOCATION/bin/python -# copy over sqlite3 headers -cp "/usr/include/sqlite3"* "$LOCATION/include/" +# # copy over sqlite3 headers +# cp "/usr/include/sqlite3"* "$LOCATION/include/" -# create generic .so symlink against stack image -ln -s "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0" "$LOCATION/lib/libsqlite3.so" +# # create generic .so symlink against stack image +# ln -s "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0" "$LOCATION/lib/libsqlite3.so" From aecdeddc322fd8da08dc227f74d899e23c509564 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Wed, 28 Aug 2019 17:21:42 -0700 Subject: [PATCH 10/23] clean up rebase --- .gitignore | 2 ++ builds/cedar-14.Dockerfile~HEAD | 15 --------------- ...entral location, add all stacks, update README | 14 -------------- ...ku-18.Dockerfile~HEAD => heroku-18.Dockerfile} | 0 ...entral location, add all stacks, update README | 15 --------------- 5 files changed, 2 insertions(+), 44 deletions(-) delete mode 100644 builds/cedar-14.Dockerfile~HEAD delete mode 100644 builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README rename builds/{heroku-18.Dockerfile~HEAD => heroku-18.Dockerfile} (100%) delete mode 100644 builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README diff --git a/.gitignore b/.gitignore index da8ab48..ea39393 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ repos/* #Venv buildpack/* + +builds/dockerenv diff --git a/builds/cedar-14.Dockerfile~HEAD b/builds/cedar-14.Dockerfile~HEAD deleted file mode 100644 index ea11542..0000000 --- a/builds/cedar-14.Dockerfile~HEAD +++ /dev/null @@ -1,15 +0,0 @@ -FROM heroku/cedar:14 - -WORKDIR /app -ENV WORKSPACE_DIR="/app/builds" \ - S3_BUCKET="lang-python" \ - S3_PREFIX="cedar-14/" \ - DEBIAN_FRONTEND=noninteractive \ - STACK="cedar-14" - -RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* - -COPY requirements.txt /app/ -RUN pip install -r /app/requirements.txt - -COPY . /app diff --git a/builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README b/builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README deleted file mode 100644 index f82fa16..0000000 --- a/builds/cedar-14.Dockerfile~move Dockerfiles to central location, add all stacks, update README +++ /dev/null @@ -1,14 +0,0 @@ -FROM heroku/cedar:14 - -WORKDIR /app -ENV WORKSPACE_DIR="/app/builds" \ - S3_BUCKET="lang-python" \ - S3_PREFIX="cedar-14/" \ - STACK="cedar-14" - -RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* - -COPY requirements.txt /app/ -RUN pip install -r /app/requirements.txt - -COPY . /app diff --git a/builds/heroku-18.Dockerfile~HEAD b/builds/heroku-18.Dockerfile similarity index 100% rename from builds/heroku-18.Dockerfile~HEAD rename to builds/heroku-18.Dockerfile diff --git a/builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README b/builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README deleted file mode 100644 index 6288ac0..0000000 --- a/builds/heroku-18.Dockerfile~move Dockerfiles to central location, add all stacks, update README +++ /dev/null @@ -1,15 +0,0 @@ -FROM heroku/heroku:18-build - -WORKDIR /app -ENV WORKSPACE_DIR="/app/builds" \ - S3_BUCKET="lang-python" \ - S3_PREFIX="heroku-18/" \ - DEBIAN_FRONTEND=noninteractive \ - STACK="heroku-18" - -RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* - -COPY requirements.txt /app/ -RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt - -COPY . /app From a724dfc62ebac9ade3e10adb9e6a1d1c5157b62f Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Thu, 29 Aug 2019 15:08:54 -0700 Subject: [PATCH 11/23] Add new formula for python3 and latest 3.6 and 3.7 patches --- builds/runtimes/python-3.6.9 | 0 builds/runtimes/python-3.7.4 | 5 +++++ builds/runtimes/python3 | 17 ++++++++++------- 3 files changed, 15 insertions(+), 7 deletions(-) mode change 100644 => 100755 builds/runtimes/python-3.6.9 create mode 100755 builds/runtimes/python-3.7.4 mode change 100644 => 100755 builds/runtimes/python3 diff --git a/builds/runtimes/python-3.6.9 b/builds/runtimes/python-3.6.9 old mode 100644 new mode 100755 diff --git a/builds/runtimes/python-3.7.4 b/builds/runtimes/python-3.7.4 new file mode 100755 index 0000000..dea889d --- /dev/null +++ b/builds/runtimes/python-3.7.4 @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ +# Build Deps: libraries/sqlite + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python3 b/builds/runtimes/python3 old mode 100644 new mode 100755 index 730e874..c0712c8 --- a/builds/runtimes/python3 +++ b/builds/runtimes/python3 @@ -6,15 +6,16 @@ set -o pipefail set -eu OUT_PREFIX=$1 +BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" +export BIN_DIR dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 dep_name=$(basename $BASH_SOURCE) # this is us -dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name -dep_package=$(echo $dep_name | tr p P)-${dep_version} # it's always "Python-…" -dep_dirname=${dep_package}-linux64 # this is uppercased -dep_archive_name=${dep_dirname}.tar.bz2 # this is uppercased too -dep_url=https://python.org/ftp/python/${dep_version}/${dep_package}.tgz +BASE=${dep_formula##*/} +python_version=${BASE^} +version_number=$(echo "$python_version" | cut -d- -f2) +dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz # echo "Installing necesary sqlite headers" # # install the necessary sqlite headers @@ -30,10 +31,12 @@ echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 echo "Building Python 3..." +echo "Pulling from source: ${dep_url}" -curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name -mv "${dep_package}" src +curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}" # --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name +echo "`ls`" +mv "${OUT_PREFIX}/${python_version}" src cd src ./configure --prefix=$OUT_PREFIX --with-ensurepip=no From b5fa65d96c38faae9206292e2f48d9f76edd10b5 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Thu, 29 Aug 2019 15:10:19 -0700 Subject: [PATCH 12/23] update Makefile to match new directory structure --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4123d78..ab2be13 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-heroku-18: buildenv-heroku-16: @echo "Creating build environment (heroku-16)..." @echo - @docker build --pull -t python-buildenv-heroku-16 . + @docker build --pull -f $(shell pwd)/builds/heroku-16.Dockerfile -t python-buildenv-heroku-16 . @echo @echo "Usage..." @echo @@ -34,7 +34,7 @@ buildenv-heroku-16: buildenv-heroku-18: @echo "Creating build environment (heroku-18)..." @echo - @docker build --pull -f Dockerfile.heroku-18 -t python-buildenv-heroku-18 . + @docker build --pull -f $(shell pwd)/builds/heroku-18.Dockerfile -t python-buildenv-heroku-18 . @echo @echo "Usage..." @echo From 12279e69d527f50de9b6d0b572c9371f6717121d Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Thu, 29 Aug 2019 15:28:07 -0700 Subject: [PATCH 13/23] Clean up comments from python3 --- builds/runtimes/python3 | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/builds/runtimes/python3 b/builds/runtimes/python3 index c0712c8..af93881 100755 --- a/builds/runtimes/python3 +++ b/builds/runtimes/python3 @@ -17,12 +17,6 @@ python_version=${BASE^} version_number=$(echo "$python_version" | cut -d- -f2) dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz -# echo "Installing necesary sqlite headers" -# # install the necessary sqlite headers -# apt-get update -# apt-get install -y libsqlite3-dev - -# Old way of installing sqlite # shellcheck source=bin/utils source "$BIN_DIR/steps/sqlite3" @@ -34,8 +28,7 @@ echo "Building Python 3..." echo "Pulling from source: ${dep_url}" -curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}" # --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name -echo "`ls`" +curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}" mv "${OUT_PREFIX}/${python_version}" src cd src @@ -51,9 +44,3 @@ find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm - LOCATION=${OUT_PREFIX%?} ln $LOCATION/bin/python3 $LOCATION/bin/python - -# # copy over sqlite3 headers -# cp "/usr/include/sqlite3"* "$LOCATION/include/" - -# # create generic .so symlink against stack image -# ln -s "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0" "$LOCATION/lib/libsqlite3.so" From 385296c270817eeaaf706b9b27963146f37aa551 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Thu, 29 Aug 2019 15:31:26 -0700 Subject: [PATCH 14/23] Add support for testing binaries in staging/ on S3 This will allow for faster and more transparent deploys for Python and other built dependencies. --- .gitignore | 3 ++- Makefile | 8 ++++++++ bin/compile | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ea39393..1ffa3cb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ repos/* #Venv buildpack/* -builds/dockerenv +builds/dockerenv.staging* +builds/dockerenv.production diff --git a/Makefile b/Makefile index ab2be13..6a09cb0 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,14 @@ check: @shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python @shellcheck -x bin/steps/hooks/* +test-staging: + @echo "Running tests against staged binaries..." + @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" -e "TEST_STAGED_S3_BINARIES=1" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' + @echo "" + @echo "Running tests in docker (heroku-18)..." + @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" -e "TEST_STAGED_S3_BINARIES=1" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' + @echo "" + test-heroku-16: @echo "Running tests in docker (heroku-16)..." @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' diff --git a/bin/compile b/bin/compile index dfc81d1..1d70e67 100755 --- a/bin/compile +++ b/bin/compile @@ -43,6 +43,13 @@ VENDOR_URL="https://lang-python.s3.amazonaws.com/$STACK" if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then VENDOR_URL="$BUILDPACK_VENDOR_URL" fi + +# Build binaries (python runtimes, etc) in the /staging/ directory on S3. Test +these binaries for compatibility when passed this environment variable. +if [ "$TEST_STAGED_S3_BINARIES" == "1" ]; then + echo "Testing staged binaries..." + VENDOR_URL="https://lang-python.s3.amazonaws.com/staging/$STACK" +fi export VENDOR_URL # Default Python Versions From 707ed848bbf1fbde1100f03d46274b84e7590021 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Thu, 29 Aug 2019 15:58:12 -0700 Subject: [PATCH 15/23] fix typo in compile --- bin/compile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 1d70e67..9d0453a 100755 --- a/bin/compile +++ b/bin/compile @@ -45,7 +45,7 @@ if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then fi # Build binaries (python runtimes, etc) in the /staging/ directory on S3. Test -these binaries for compatibility when passed this environment variable. +# these binaries for compatibility when passed this environment variable. if [ "$TEST_STAGED_S3_BINARIES" == "1" ]; then echo "Testing staged binaries..." VENDOR_URL="https://lang-python.s3.amazonaws.com/staging/$STACK" From a8da6e4ed55a8a8f847b0aa65049ab62da52cee4 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Fri, 30 Aug 2019 09:17:44 -0700 Subject: [PATCH 16/23] add cedar-14 back in after clarification on support policy Heroku will continue to build binaries as long as stacks receive updates, even after EOL --- builds/README.md | 3 +-- builds/cedar-14.Dockerfile | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 builds/cedar-14.Dockerfile diff --git a/builds/README.md b/builds/README.md index 7b39fde..200893a 100644 --- a/builds/README.md +++ b/builds/README.md @@ -4,6 +4,7 @@ **After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack: + $ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile . $ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile . $ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile . @@ -31,8 +32,6 @@ Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCK If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file. -If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file. - #### Passing credentials explicitly docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash diff --git a/builds/cedar-14.Dockerfile b/builds/cedar-14.Dockerfile new file mode 100644 index 0000000..982a9d7 --- /dev/null +++ b/builds/cedar-14.Dockerfile @@ -0,0 +1,15 @@ +FROM heroku/cedar:14 + +WORKDIR /app +ENV WORKSPACE_DIR="/app/builds" \ + S3_BUCKET="lang-python" \ + S3_PREFIX="cedar-14/" \ + DEBIAN_FRONTEND=noninteractive \ + STACK="cedar-14" + +RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/ +RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt + +COPY . /app From b6e2739fb5ed74b4004dfc898a8a4b7e364bfde6 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Thu, 5 Sep 2019 08:54:01 -0700 Subject: [PATCH 17/23] add Cedar 14 environment back for tests and binaries --- Makefile | 12 ++++++++++-- builds/cedar-14.Dockerfile | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6a09cb0..53f9a97 100644 --- a/Makefile +++ b/Makefile @@ -9,13 +9,21 @@ check: @shellcheck -x bin/steps/hooks/* test-staging: - @echo "Running tests against staged binaries..." + @echo "Running tests against staged binaries (cedar-14)..." + @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" -e "TEST_STAGED_S3_BINARIES=1" heroku/cedar-14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' + @echo "" + @echo "Running tests against staged binaries (heroku-16)..." @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" -e "TEST_STAGED_S3_BINARIES=1" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' @echo "" - @echo "Running tests in docker (heroku-18)..." + @echo "Running tests against staged binaries (heroku-18)..." @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" -e "TEST_STAGED_S3_BINARIES=1" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' @echo "" +test-cedar-14: + @echo "Running tests in docker (cedar-14)..." + @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar-14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' + @echo "" + test-heroku-16: @echo "Running tests in docker (heroku-16)..." @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' diff --git a/builds/cedar-14.Dockerfile b/builds/cedar-14.Dockerfile index 982a9d7..ea11542 100644 --- a/builds/cedar-14.Dockerfile +++ b/builds/cedar-14.Dockerfile @@ -10,6 +10,6 @@ ENV WORKSPACE_DIR="/app/builds" \ RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* COPY requirements.txt /app/ -RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt +RUN pip install -r /app/requirements.txt COPY . /app From 9031333ef4fb5e00fe183ef4183d19fc8a3e29e9 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Thu, 5 Sep 2019 16:48:51 -0700 Subject: [PATCH 18/23] change travis to bionic --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a161820..7876b7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: ruby -dist: trusty +dist: bionic sudo: required rvm: From 158a3e28a83a0c12d9f60b93a5b82d4bcdcdc95b Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Tue, 10 Sep 2019 18:26:14 -0700 Subject: [PATCH 19/23] update travis to match dockerfile locations --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7876b7d..c00ce25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,11 @@ before_script: - bundle exec hatchet ci:setup script: - - docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/Dockerfile . + - docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile . + - docker run --rm -e "STACK=cedar-14" heroku-python-build-heroku-16 bash $TESTFOLDER + - docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile . - docker run --rm -e "STACK=heroku-16" heroku-python-build-heroku-16 bash $TESTFOLDER - - docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/Dockerfile.heroku-18 . + - docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile . - docker run --rm -e "STACK=heroku-18" heroku-python-build-heroku-18 bash $TESTFOLDER jobs: From 06694bba18532a13de4671c34e7babc4cf0b8743 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Tue, 10 Sep 2019 18:29:04 -0700 Subject: [PATCH 20/23] simplify pr for review, pull out staged binaries tooling --- Makefile | 11 ----------- bin/compile | 8 -------- 2 files changed, 19 deletions(-) diff --git a/Makefile b/Makefile index 53f9a97..abc769b 100644 --- a/Makefile +++ b/Makefile @@ -8,17 +8,6 @@ check: @shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python @shellcheck -x bin/steps/hooks/* -test-staging: - @echo "Running tests against staged binaries (cedar-14)..." - @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" -e "TEST_STAGED_S3_BINARIES=1" heroku/cedar-14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' - @echo "" - @echo "Running tests against staged binaries (heroku-16)..." - @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" -e "TEST_STAGED_S3_BINARIES=1" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' - @echo "" - @echo "Running tests against staged binaries (heroku-18)..." - @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" -e "TEST_STAGED_S3_BINARIES=1" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' - @echo "" - test-cedar-14: @echo "Running tests in docker (cedar-14)..." @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar-14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' diff --git a/bin/compile b/bin/compile index 9d0453a..6e49277 100755 --- a/bin/compile +++ b/bin/compile @@ -44,14 +44,6 @@ if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then VENDOR_URL="$BUILDPACK_VENDOR_URL" fi -# Build binaries (python runtimes, etc) in the /staging/ directory on S3. Test -# these binaries for compatibility when passed this environment variable. -if [ "$TEST_STAGED_S3_BINARIES" == "1" ]; then - echo "Testing staged binaries..." - VENDOR_URL="https://lang-python.s3.amazonaws.com/staging/$STACK" -fi -export VENDOR_URL - # Default Python Versions # shellcheck source=bin/default_pythons source "$BIN_DIR/default_pythons" From 783c2e2329c19af77a93c7612f4c4988bb80f8f0 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Tue, 10 Sep 2019 18:35:54 -0700 Subject: [PATCH 21/23] ensure and add comments to base python formulas --- builds/runtimes/python2 | 35 +++++++++++++++++++++++++++++++++++ builds/runtimes/python3 | 9 ++++----- 2 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 builds/runtimes/python2 diff --git a/builds/runtimes/python2 b/builds/runtimes/python2 new file mode 100644 index 0000000..5794991 --- /dev/null +++ b/builds/runtimes/python2 @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +OUT_PREFIX=$1 +BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" +export BIN_DIR + +# Orient ourselves and build download link +dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 +BASE=${dep_formula##*/} # this gives us relative path +python_version=${BASE^} # this gives us only the filename with version number +version_number=$(echo "$python_version" | cut -d- -f2) # this returns just X.X.X +dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz + +# shellcheck source=bin/utils +source "$BIN_DIR/steps/sqlite3" + +sqlite3_version +echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 + +echo "Building Python 3..." +echo "Pulling from source: ${dep_url}" + +curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}" +mv "${OUT_PREFIX}/${python_version}" src +cd src + +./configure --prefix=$OUT_PREFIX --enable-unicode=ucs4 --with-ensurepip=no +make +make install + +# Remove unneeded test directories, similar to the official Docker Python images: +# https://github.com/docker-library/python +find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' + diff --git a/builds/runtimes/python3 b/builds/runtimes/python3 index af93881..79f4421 100755 --- a/builds/runtimes/python3 +++ b/builds/runtimes/python3 @@ -9,12 +9,11 @@ OUT_PREFIX=$1 BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" export BIN_DIR +# Orient ourselves and build download link dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 -dep_name=$(basename $BASH_SOURCE) # this is us - -BASE=${dep_formula##*/} -python_version=${BASE^} -version_number=$(echo "$python_version" | cut -d- -f2) +BASE=${dep_formula##*/} # this gives us relative path +python_version=${BASE^} # this gives us only the filename with version number +version_number=$(echo "$python_version" | cut -d- -f2) # this returns just X.X.X dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz # shellcheck source=bin/utils From a239fde78c5aa11d1ae706c46432b17a80dd6b8d Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Tue, 10 Sep 2019 18:50:50 -0700 Subject: [PATCH 22/23] clarify naming for travis docker tags and correct typo --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index c00ce25..f8e7e93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,12 +9,12 @@ before_script: - bundle exec hatchet ci:setup script: - - docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile . - - docker run --rm -e "STACK=cedar-14" heroku-python-build-heroku-16 bash $TESTFOLDER - - docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile . - - docker run --rm -e "STACK=heroku-16" heroku-python-build-heroku-16 bash $TESTFOLDER - - docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile . - - docker run --rm -e "STACK=heroku-18" heroku-python-build-heroku-18 bash $TESTFOLDER + - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile . + - docker run --rm -e "STACK=cedar-14" travis-build-cedar-14 bash $TESTFOLDER + - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile . + - docker run --rm -e "STACK=heroku-16" travis-build-heroku-16 bash $TESTFOLDER + - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile . + - docker run --rm -e "STACK=heroku-18" travis-build-heroku-18 bash $TESTFOLDER jobs: include: From 006ea77641d6212a35ef3ece8562082639efbb66 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Wed, 11 Sep 2019 11:57:31 -0700 Subject: [PATCH 23/23] address issues found in review --- .travis.yml | 2 +- bin/compile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f8e7e93..d5449a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: ruby -dist: bionic +dist: trusty sudo: required rvm: diff --git a/bin/compile b/bin/compile index 6e49277..dfc81d1 100755 --- a/bin/compile +++ b/bin/compile @@ -43,6 +43,7 @@ VENDOR_URL="https://lang-python.s3.amazonaws.com/$STACK" if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then VENDOR_URL="$BUILDPACK_VENDOR_URL" fi +export VENDOR_URL # Default Python Versions # shellcheck source=bin/default_pythons