kennethreitz/heroku-buildpack-python
1
0
Fork 0
mirror of https://github.com/kennethreitz/heroku-buildpack-python.git synced 2026-06-05 23:10:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Simplify binary build process and documentation (#1083)

Browse Source 
* Moves all manual build steps to make targets, to simplify
  the commands run, and reduce chance for error.
* Removes the need to remember to rebuild the builder
  image by building it automatically prior to launching.
* Adds a new make target for deploying multiple runtime
  versions at once to speed up the common case.
* Reduces repetition/superfluous content in documentation.
* Removes unused `S3_REGION` from `dockerenv.default`
  (the contents of S3 buckets inherit the region of the bucket).
* Documents build dependencies in `requirements.txt`.

Closes @W-8119717@.

[skip changelog]
This commit is contained in:
Ed Morley
2020-09-29 13:40:56 +01:00
committed by GitHub
parent 0020aae078
commit 838f4c125b
4 changed files with 104 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+33 -25
View File
@@ -1,15 +1,18 @@
# These targets are not files
.PHONY: check test buildenv-heroku-16 buildenv-heroku-18 tools
.PHONY: check test builder-image buildenv deploy-runtimes tools
STACK ?= heroku-18
STACKS ?= cedar-14 heroku-16 heroku-18
TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
ENV_FILE ?= builds/dockerenv.default
BUILDER_IMAGE_PREFIX := heroku-python-build
ifeq ($(STACK),cedar-14)
# Cedar-14 doesn't have a build image varient.
IMAGE_TAG := heroku/cedar:14
STACK_IMAGE_TAG := heroku/cedar:14
else
# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
STACK_IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
endif
check:
@@ -19,35 +22,40 @@ check:
test:
@echo "Running tests using: STACK=$(STACK) TEST_CMD='$(TEST_CMD)'"
@echo ""
@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
@echo ""
buildenv-heroku-16:
@echo "Creating build environment (heroku-16)..."
@echo
@docker build --pull -f $(shell pwd)/builds/heroku-16.Dockerfile -t python-buildenv-heroku-16 .
@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(STACK_IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
@echo
builder-image:
@echo "Generating binary builder image for $(STACK)..."
@echo
@docker build --pull -f builds/$(STACK).Dockerfile -t "$(BUILDER_IMAGE_PREFIX)-$(STACK)" .
@echo
buildenv: builder-image
@echo "Starting build environment for $(STACK)..."
@echo
@echo "Usage..."
@echo
@echo " $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar # Optional unless deploying"
@echo " $$ bob build runtimes/python-2.7.13"
@echo " $$ bob deploy runtimes/python-2.7.13"
@echo " $$ bob build runtimes/python-X.Y.Z"
@echo
@docker run -it --rm python-buildenv-heroku-16
@docker run --rm -it --env-file="$(ENV_FILE)" -v $(PWD)/builds:/app/builds "$(BUILDER_IMAGE_PREFIX)-$(STACK)" bash
buildenv-heroku-18:
@echo "Creating build environment (heroku-18)..."
deploy-runtimes:
ifndef RUNTIMES
$(error No runtimes specified! Use: "make deploy-runtimes RUNTIMES='python-X.Y.Z ...' [STACKS='heroku-18 ...'] [ENV_FILE=...]")
endif
@echo "Using: RUNTIMES='$(RUNTIMES)' STACKS='$(STACKS)' ENV_FILE='$(ENV_FILE)'"
@echo
@docker build --pull -f $(shell pwd)/builds/heroku-18.Dockerfile -t python-buildenv-heroku-18 .
@echo
@echo "Usage..."
@echo
@echo " $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar # Optional unless deploying"
@echo " $$ bob build runtimes/python-2.7.13"
@echo " $$ bob deploy runtimes/python-2.7.13"
@echo
@docker run -it --rm python-buildenv-heroku-18
@set -eu; for stack in $(STACKS); do \
$(MAKE) builder-image STACK=$${stack}; \
for runtime in $(RUNTIMES); do \
echo "Generating/deploying $${runtime} for $${stack}..."; \
echo; \
docker run --rm -it --env-file="$(ENV_FILE)" "$(BUILDER_IMAGE_PREFIX)-$${stack}" bob deploy "runtimes/$${runtime}"; \
echo; \
done; \
done
tools:
git clone https://github.com/kennethreitz/pip-pop.git
builds/README.md
+61 -38
View File
@@ -1,57 +1,80 @@
# Python Buildpack Binaries
## Building the Docker Images
**After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack:
$ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
$ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
$ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
## Using the Image
You can e.g. `bash` into each of the images you built using their tag:
docker run --rm -ti heroku-python-build-cedar-14 bash
docker run --rm -ti heroku-python-build-heroku-16 bash
docker run --rm -ti heroku-python-build-heroku-18 bash
You then have a shell where you can run `bob build`, `bob deploy`, and so forth. You can of course also invoke these programs directly with `docker run`:
docker run --rm -ti heroku-python-build-heroku-18 bob build runtimes/python-2.7.15
In order to `bob deploy`, AWS credentials must be set up, as well as name and prefix of your custom S3 bucket (unless you're deploying to the Heroku production buckets that are pre-defined in each `Dockerfile`); see next section for details.
The binaries for this buildpack are built in Docker containers based on the Heroku stack image.
## Configuration
File `dockerenv.default` contains a list of required env vars; most of these have default values defined in `Dockerfile`. You can copy this file to a location outside the buildpack and modify it with the values you desire and pass its location with `--env-file`, or pass the env vars to `docker run` using `--env`.
In order to publish binaries AWS credentials must be passed to the build container.
If you are testing only the build (ie: `bob build`), these are optional.
Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCKET`, `S3_PREFIX`, and `S3_REGION`. If you're building your own packages, you'll likely want to change `S3_BUCKET` and `S3_PREFIX` to match your info. Instead of setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` into that file, you may also pass them to `docker run` through the environment, or explicitly using `--env`, in order to prevent accidental commits of credentials.
In addition, unless you are building the official binaries for Heroku (which use the defaults
specified in each `Dockerfile`), you will need to override `S3_BUCKET` and `S3_PREFIX` to
match your own S3 bucket/use case.
### Passing AWS credentials to the container
If you only need to set AWS credentials, you can do so by setting the environment variables
`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` before calling the make commands.
If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file.
For example:
#### Passing credentials explicitly
```bash
set +o history # Disable bash history
export AWS_ACCESS_KEY_ID=...
export AWS_SECRET_ACCESS_KEY=...
set -o history # Re-enable bash history
make ...
```
docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash
If you need to override the default S3 bucket, or would prefer not to use credentials via
environment variables, then you need to instead use a Docker env file like so:
#### Passing credentials through the environment
1. Copy the `builds/dockerenv.default` env file to a location outside the buildpack repository.
2. Edit the new file, adding at a minimum the values for the variables
`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (see Docker
[env-file documentation](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file)).
3. Pass the path of the file to the make commands using `ENV_FILE`. For example:
The two environment variables `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY` are defined in `builds/dockerenv.default`, without values. This will cause Docker to "forward" values for these variables from the current environment, so you can pass them in:
```bash
make ... ENV_FILE=~/.dockerenv.python-buildpack
```
AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
## Launching an interactive build environment
or
To start an interactive version of the build environment (ideal for development) use the
`buildenv` make target, passing in the desired `STACK` name. For example:
export AWS_ACCESS_KEY_ID=...
export AWS_SECRET_ACCESS_KEY=...
docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
```bash
make buildenv STACK=heroku-18
```
#### Passing credentials through a separate env file
This will create the builder docker image based on the latest image for that stack, and
then start a bash shell where you can run `bob build`, `bob deploy`, and so forth.
This method is the easiest for users who want to build packages in their own S3 bucket, as they will have to adjust the `S3_BUCKET` and `S3_PREFIX` environment variable values anyway from their default values.
The `builds/` directory is bind-mounted into the running container, so local build formula
changes will appear there immediately without the need to rebuild the image.
For this method, it is important to keep the credentials file in a location outside the buildpack, so that your credentials aren't accidentally committed. Copy `builds/dockerenv.default` **to a safe location outside the buildpack directory**, and insert your values for `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY`.
## Bulk deploying runtimes
docker run --rm -ti --env-file=../SOMEPATHOUTSIDE/s3.env heroku-python-build-heroku-18 bash
When a new Python version is released, binaries have to be generated for multiple stacks.
To automate this, use the `deploy-runtimes` make target, which will ensure the builder
image is up to date, and then run `bob deploy` for each runtime-stack combination.
The build formula name(s) are passed using `RUNTIMES`, like so:
```bash
make deploy-runtimes RUNTIMES='python-X.Y.Z'
```
By default this will deploy to all supported stacks (see `STACKS` in `Makefile`),
but this can be overridden using `STACKS`:
```bash
make deploy-runtimes RUNTIMES='python-X.Y.Z' STACKS='heroku-16 heroku-18'
```
Multiple runtimes can also be specified (useful for when adding a new stack), like so:
```bash
make deploy-runtimes RUNTIMES='python-A.B.C python-X.Y.Z' STACKS='heroku-20'
```
Note: Both `RUNTIMES` and `STACKS` are space delimited.
builds/dockerenv.default
+6 -3
View File
@@ -1,5 +1,8 @@
# Since no values are specified here, these variables will be read from the environment at run time:
# https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file
AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY
S3_BUCKET
S3_PREFIX
S3_REGION
# Uncomment these if you need to override the default S3 bucket and/or path prefixes.
# S3_BUCKET
# S3_PREFIX
requirements.txt
+4 -1
View File
@@ -1,3 +1,6 @@
docopt==0.6.2
# Dependencies for generating/publishing Python binaries.
bob-builder==0.0.18
# Sub-dependencies of bob-builder.
boto==2.49.0
docopt==0.6.2