Merge pull request #848 from heroku/cleanup-output

download get-pip silently

.gitignore

@@ -7,3 +7,6 @@ repos/*
 
 #Venv
 buildpack/*
+
+builds/dockerenv.staging*
+builds/dockerenv.production

.pre-commit-config.yaml

@@ -1,5 +1,7 @@
 repos:
--   repo: git://github.com/detailyang/pre-commit-shell
+-   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: 1.0.4
+    rev: v2.3.0
     hooks:
-    - id: shell-lint
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace

.travis.yml

@@ -1,35 +1,39 @@
 language: ruby
 dist: trusty
 sudo: required
+
 rvm:
 - 2.4.4
 before_script:
   - gem install bundler -v 1.16.2
   - bundle exec hatchet ci:setup
+
+script:
+  - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
+  - docker run --rm -e "STACK=cedar-14" travis-build-cedar-14 bash $TESTFOLDER
+  - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
+  - docker run --rm -e "STACK=heroku-16" travis-build-heroku-16 bash $TESTFOLDER
+  - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
+  - docker run --rm -e "STACK=heroku-18" travis-build-heroku-18 bash $TESTFOLDER
+
 jobs:
   include:
   - stage: Bash linting (shellcheck)
     sudo: false
     script: make check
-  - stage: Stack Unit Tests
-    services: docker
-    env: STACK=heroku-18
-    script: "./tests.sh"
-  - stage: Stack Unit Tests
-    services: docker
-    env: STACK=heroku-16
-    script: "./tests.sh"
-  - stage: Stack Unit Tests
-    services: docker
-    env: STACK=cedar-14
-    script: "./tests.sh"
   - stage: Hatchet Integration
+    if: branch = master
+    name: Run Hatchet
     script: "bundle exec rspec"
 env:
+  matrix:
+    - TESTFOLDER=test/run-deps
+    - TESTFOLDER=test/run-versions
+    - TESTFOLDER=test/run-features
   global:
-  - HATCHET_RETRIES=3
+    - HATCHET_RETRIES=3
-  - IS_RUNNING_ON_CI=true
+    - IS_RUNNING_ON_CI=true
-  - HATCHET_APP_LIMIT=5
+    - HATCHET_APP_LIMIT=5
-  - HATCHET_DEPLOY_STRATEGY=git
+    - HATCHET_DEPLOY_STRATEGY=git
-  - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
+    - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
-  - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=
+    - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Python Buildpack Changelog
 
+# Master
+
+--------------------------------------------------------------------------------
+
+# 156 (2019-09-12)
+
+- Python 3.6.9 and 3.7.4 now available.
+
+- Move get-pip utility to S3
+- Build utility and documentation updates
+- Bump Hatchet tests to point at new default python version.
+
+# 155 (2019-08-22)
+
+add docs and make target for heroku-18 bob builds
+
 # 154 (2019-07-17)
 
 Fix python 3.5.7 formula actually building 3.7.2

Makefile

@@ -1,7 +1,7 @@
 # These targets are not files
 .PHONY: tests
 
-test: test-heroku-18 test-heroku-16
+test: test-heroku-18 test-heroku-16 test-cedar-14
 
 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
@@ -10,23 +10,23 @@ check:
 
 test-cedar-14:
 	@echo "Running tests in docker (cedar-14)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
 
 test-heroku-16:
 	@echo "Running tests in docker (heroku-16)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
 
 test-heroku-18:
 	@echo "Running tests in docker (heroku-18)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
 
 buildenv-heroku-16:
 	@echo "Creating build environment (heroku-16)..."
 	@echo
-	@docker build --pull -t python-buildenv-heroku-16 .
+	@docker build --pull -f $(shell pwd)/builds/heroku-16.Dockerfile -t python-buildenv-heroku-16 .
 	@echo
 	@echo "Usage..."
 	@echo
@@ -36,6 +36,19 @@ buildenv-heroku-16:
 	@echo
 	@docker run -it --rm python-buildenv-heroku-16
 
+buildenv-heroku-18:
+	@echo "Creating build environment (heroku-18)..."
+	@echo
+	@docker build --pull -f $(shell pwd)/builds/heroku-18.Dockerfile -t python-buildenv-heroku-18 .
+	@echo
+	@echo "Usage..."
+	@echo
+	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
+	@echo "  $$ bob build runtimes/python-2.7.13"
+	@echo "  $$ bob deploy runtimes/python-2.7.13"
+	@echo
+	@docker run -it --rm python-buildenv-heroku-18
+
 tools:
 	git clone https://github.com/kennethreitz/pip-pop.git
 	mv pip-pop/bin/* vendor/pip-pop/

README.md

@@ -26,7 +26,7 @@ remote: Compressing source files... done.
 remote: Building source:
 remote:
 remote: -----> Python app detected
-remote: -----> Installing python-3.7.1
+remote: -----> Installing python-3.7.4
 remote: -----> Installing pip
 remote: -----> Installing SQLite3
 remote: -----> Installing requirements with pip
@@ -44,7 +44,7 @@ A `requirements.txt` must be present at the root of your application's repositor
 
 To specify your python version, you also need a `runtime.txt` file - unless you are using the default Python runtime version.
 
-Current default Python Runtime: Python 3.6.7
+Current default Python Runtime: Python 3.6.9
 
 Alternatively, you can provide a `setup.py` file, or a `Pipfile`. Using `Pipenv` will generate `runtime.txt` based on `python-version` at build time.
 
@@ -61,9 +61,9 @@ Specify a Python Runtime
 
 Supported runtime options include:
 
-- `python-3.7.1`
+- `python-3.7.4`
-- `python-3.6.7`
+- `python-3.6.9`
-- `python-2.7.15`
+- `python-2.7.16`
 
 ## Tests
 

bin/default_pythons

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-DEFAULT_PYTHON_VERSION="python-3.6.8"
+DEFAULT_PYTHON_VERSION="python-3.6.9"
-LATEST_36="python-3.6.8"
+LATEST_36="python-3.6.9"
-LATEST_37="python-3.7.3"
+LATEST_37="python-3.7.4"
 LATEST_35="python-3.5.7"
 LATEST_34="python-3.4.10"
 LATEST_27="python-2.7.16"

bin/steps/pipenv

@@ -26,7 +26,7 @@ fi
 
 
 if [ ! "$SKIP_PIPENV_INSTALL" ]; then
-    # Pipenv support (Generate requriements.txt with pipenv).
+    # Pipenv support (Generate requirements.txt with pipenv).
     if [[ -f Pipfile ]]; then
         # Measure that we're using Pipenv.
         mcount "tool.pipenv"

bin/steps/python

@@ -97,6 +97,15 @@ if [ ! "$SKIP_INSTALL" ]; then
   hash -r
 fi
 
+# Heroku uses the get-pip utility maintained by the Python community to vendor Pip.
+# https://github.com/pypa/get-pip
+GETPIP="https://lang-python.s3.amazonaws.com/etc/get-pip.py"
+
+if ! curl -s "${GETPIP}" -o "$ROOT_DIR/get-pip.py" &> /dev/null; then
+    mcount "failure.python.get-pip"
+    echo "Failed to pull down get-pip"
+    exit 1
+fi
 
 # If Pip isn't up to date:
 if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
@@ -107,7 +116,7 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
   rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
   rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
 
-  /app/.heroku/python/bin/python "$ROOT_DIR/vendor/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null
+  /app/.heroku/python/bin/python "$ROOT_DIR/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null
   /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
 fi
 

builds/README.md

@@ -1,41 +1,57 @@
 # Python Buildpack Binaries
 
-For Cedar-14 stack
+## Building the Docker Images
-------------------
 
-To get started with it, create an app on Heroku inside a clone of this repository, and set your S3 config vars:
+**After every change to your formulae, perform the following** from the root of the Git repository (not from `builds/`) to rebuild the images for each stack:
 
-    $ heroku create --buildpack https://github.com/heroku/heroku-buildpack-python#not-heroku
+    $ docker build --pull --tag heroku-python-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
-    $ heroku config:set WORKSPACE_DIR=builds
+    $ docker build --pull --tag heroku-python-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-    $ heroku config:set AWS_ACCESS_KEY_ID=<your_aws_key>
+    $ docker build --pull --tag heroku-python-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-    $ heroku config:set AWS_SECRET_ACCESS_KEY=<your_aws_secret>
-    $ heroku config:set S3_BUCKET=<your_s3_bucket_name>
 
+## Using the Image
 
-Then, shell into an instance and run a build by giving the name of the formula inside `builds`:
+You can e.g. `bash` into each of the images you built using their tag:
 
-    $ heroku run bash
+    docker run --rm -ti heroku-python-build-cedar-14 bash
-    Running `bash` attached to terminal... up, run.6880
+    docker run --rm -ti heroku-python-build-heroku-16 bash
-    ~ $ bob build runtimes/python-2.7.6
+    docker run --rm -ti heroku-python-build-heroku-18 bash
 
-    Fetching dependencies... found 2:
+You then have a shell where you can run `bob build`, `bob deploy`, and so forth. You can of course also invoke these programs directly with `docker run`:
-      - libraries/sqlite
 
-    Building formula runtimes/python-2.7.6:
+    docker run --rm -ti heroku-python-build-heroku-18 bob build runtimes/python-2.7.15
-        === Building Python 2.7.6
-        Fetching Python v2.7.6 source...
-        Compiling...
 
-If this works, run `bob deploy` instead of `bob build` to have the result uploaded to S3 for you.
+In order to `bob deploy`, AWS credentials must be set up, as well as name and prefix of your custom S3 bucket (unless you're deploying to the Heroku production buckets that are pre-defined in each `Dockerfile`); see next section for details.
 
-To speed things up drastically, it'll usually be a good idea to `heroku run bash --size PX` instead.
+## Configuration
 
-For Heroku-16 stack
+File `dockerenv.default` contains a list of required env vars; most of these have default values defined in `Dockerfile`. You can copy this file to a location outside the buildpack and modify it with the values you desire and pass its location with `--env-file`, or pass the env vars to `docker run` using `--env`.
--------------------
 
-1. Ensure GNU Make and Docker are installed.
+Out of the box, each `Dockerfile` has the correct values predefined for `S3_BUCKET`, `S3_PREFIX`, and `S3_REGION`. If you're building your own packages, you'll likely want to change `S3_BUCKET` and `S3_PREFIX` to match your info. Instead of setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` into that file, you may also pass them to `docker run` through the environment, or explicitly using `--env`, in order to prevent accidental commits of credentials.
-2. From the root of the buildpack repository, run: `make buildenv-heroku-16`
-3. Follow the instructions displayed!
 
+### Passing AWS credentials to the container
 
-Enjoy :)
+If you want to deploy packages and thus need to pass `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, you can either pass them explicitly, through your environment, or through an env file.
+
+#### Passing credentials explicitly
+
+    docker run --rm -ti -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-python-build-heroku-18 bash
+
+#### Passing credentials through  the environment
+
+The two environment variables `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY` are defined in `builds/dockerenv.default`, without values. This will cause Docker to "forward" values for these variables from the current environment, so you can pass them in:
+
+    AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+
+or
+
+    export AWS_ACCESS_KEY_ID=...
+    export AWS_SECRET_ACCESS_KEY=...
+    docker run --rm -ti --env-file=builds/dockerenv.default heroku-python-build-heroku-18 bash
+
+#### Passing credentials through a separate env file
+
+This method is the easiest for users who want to build packages in their own S3 bucket, as they will have to adjust the `S3_BUCKET` and `S3_PREFIX` environment variable values anyway from their default values.
+
+For this method, it is important to keep the credentials file in a location outside the buildpack, so that your credentials aren't accidentally committed. Copy `builds/dockerenv.default` **to a safe location outside the buildpack directory**, and insert your values for `AWS_ACCESS_KEY_ID`and `AWS_SECRET_ACCESS_KEY`.
+
+    docker run --rm -ti --env-file=../SOMEPATHOUTSIDE/s3.env heroku-python-build-heroku-18 bash

builds/cedar-14.Dockerfile

@@ -1,14 +1,15 @@
-FROM heroku/heroku:18-build
+FROM heroku/cedar:14
 
 WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
     S3_BUCKET="lang-python" \
-    S3_PREFIX="heroku-18/" \
+    S3_PREFIX="cedar-14/" \
-    DEBIAN_FRONTEND=noninteractive
+    DEBIAN_FRONTEND=noninteractive \
+    STACK="cedar-14"
 
 RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
 
 COPY requirements.txt /app/
-RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+RUN pip install -r /app/requirements.txt
 
 COPY . /app

builds/dockerenv.default

@@ -0,0 +1,5 @@
+AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY
+S3_BUCKET
+S3_PREFIX
+S3_REGION

builds/heroku-16.Dockerfile

@@ -4,7 +4,8 @@ WORKDIR /app
 ENV WORKSPACE_DIR="/app/builds" \
     S3_BUCKET="lang-python" \
     S3_PREFIX="heroku-16/" \
-    DEBIAN_FRONTEND=noninteractive
+    DEBIAN_FRONTEND=noninteractive \
+    STACK="heroku-16"
 
 RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
 

builds/heroku-18.Dockerfile

@@ -0,0 +1,15 @@
+FROM heroku/heroku:18-build
+
+WORKDIR /app
+ENV WORKSPACE_DIR="/app/builds" \
+    S3_BUCKET="lang-python" \
+    S3_PREFIX="heroku-18/" \
+    DEBIAN_FRONTEND=noninteractive \
+    STACK="heroku-18"
+
+RUN apt-get update && apt-get install --no-install-recommends -y python-pip-whl=9.0.1-2 python-pip=9.0.1-2 python-setuptools python-wheel && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/
+RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+
+COPY . /app

builds/runtimes/pypy

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# fail hard
+set -o pipefail
+# fail harder
+set -eu
+
+OUT_PREFIX=$1
+
+dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
+dep_name=$(basename $BASH_SOURCE) # this is us
+dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name
+dep_package=${dep_name}2-v${dep_version} # it's always "pypy2-…"
+dep_dirname=${dep_package}-linux64
+dep_archive_name=${dep_dirname}.tar.bz2
+dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+
+echo "Building PyPy…"
+
+curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+
+ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python"

builds/runtimes/pypy-5.3.1

@@ -2,13 +2,4 @@
 # Build Path: /app/.heroku/python/
 # Build Deps: libraries/sqlite
 
-# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+source $(dirname $0)/pypy
-
-OUT_PREFIX=$1
-
-echo "Building PyPy…"
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy2-v5.3.1-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/pypy-5.6.0

@@ -2,13 +2,4 @@
 # Build Path: /app/.heroku/python/
 # Build Deps: libraries/sqlite
 
-# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+source $(dirname $0)/pypy
-
-OUT_PREFIX=$1
-
-echo "Building PyPy…"
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy2-v5.6.0-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/pypy-5.7.0

@@ -2,13 +2,4 @@
 # Build Path: /app/.heroku/python/
 # Build Deps: libraries/sqlite
 
-# NOTICE: This formula only works for the cedar-14 and heroku-16 stacks, not cedar.
+source $(dirname $0)/pypy
-
-OUT_PREFIX=$1
-
-echo "Building PyPy…"
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.0-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy2-v5.7.0-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/pypy-5.7.1

@@ -2,13 +2,4 @@
 # Build Path: /app/.heroku/python/
 # Build Deps: libraries/sqlite
 
-# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+source $(dirname $0)/pypy
-
-OUT_PREFIX=$1
-
-echo "Building PyPy…"
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy2-v5.7.1-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/pypy-5.8.0

@@ -2,11 +2,4 @@
 # Build Path: /app/.heroku/python/
 # Build Deps: libraries/sqlite
 
-OUT_PREFIX=$1
+source $(dirname $0)/pypy
-
-echo "Building PyPy…"
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy2-v5.8.0-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/pypy3

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# fail hard
+set -o pipefail
+# fail harder
+set -eu
+
+OUT_PREFIX=$1
+
+dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
+dep_name=$(basename $BASH_SOURCE) # this is us
+dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name
+dep_package=${dep_name}${dep_version_prefix:-}-v${dep_version}${dep_version_suffix:-}
+dep_dirname=${dep_package}-linux64
+dep_archive_name=${dep_dirname}.tar.bz2
+dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+
+echo "Building PyPy3…"
+
+curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+
+ln "$OUT_PREFIX/bin/pypy3" "$OUT_PREFIX/bin/python"

builds/runtimes/pypy3-2.4.0

@@ -2,13 +2,4 @@
 # Build Path: /app/.heroku/python/
 # Build Deps: libraries/sqlite
 
-# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+source $(dirname $0)/pypy3
-
-OUT_PREFIX=$1
-
-echo "Building PyPy…"
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-2.4.0-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy3-2.4.0-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python

builds/runtimes/pypy3-5.5.0

@@ -2,13 +2,7 @@
 # Build Path: /app/.heroku/python/
 # Build Deps: libraries/sqlite
 
-# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+# version pypy3-5.5.0 is really called pypy3.3-v5.5.0-alpha
-
+dep_version_prefix=".3"
-OUT_PREFIX=$1
+dep_version_suffix="-alpha"
-
+source $(dirname $0)/pypy3
-echo "Building PyPy…"
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3.3-v5.5.0-alpha-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy3-v5.5.0-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python

builds/runtimes/pypy3-5.7.1

@@ -2,11 +2,4 @@
 # Build Path: /app/.heroku/python/
 # Build Deps: libraries/sqlite
 
-OUT_PREFIX=$1
+source $(dirname $0)/pypy3
-
-echo "Building PyPy…"
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy3-v5.7.1-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python

builds/runtimes/pypy3-5.8.0

@@ -2,11 +2,4 @@
 # Build Path: /app/.heroku/python/
 # Build Deps: libraries/sqlite
 
-OUT_PREFIX=$1
+source $(dirname $0)/pypy3
-
-echo "Building PyPy…"
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.8.0-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy3-v5.8.0-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python

builds/runtimes/python-3.6.9

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+source $(dirname $0)/python3

builds/runtimes/python-3.7.4

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+source $(dirname $0)/python3

builds/runtimes/python2

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# Orient ourselves and build download link
+dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
+BASE=${dep_formula##*/} # this gives us relative path
+python_version=${BASE^} # this gives us only the filename with version number
+version_number=$(echo "$python_version" | cut -d- -f2) # this returns just X.X.X
+dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python 3..."
+echo "Pulling from source: ${dep_url}"
+
+curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}"
+mv "${OUT_PREFIX}/${python_version}" src
+cd src
+
+./configure --prefix=$OUT_PREFIX  --enable-unicode=ucs4 --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +

builds/runtimes/python3

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# fail hard
+set -o pipefail
+# fail harder
+set -eu
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# Orient ourselves and build download link
+dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
+BASE=${dep_formula##*/} # this gives us relative path
+python_version=${BASE^} # this gives us only the filename with version number
+version_number=$(echo "$python_version" | cut -d- -f2) # this returns just X.X.X
+dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python 3..."
+echo "Pulling from source: ${dep_url}"
+
+
+curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}"
+mv "${OUT_PREFIX}/${python_version}" src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

spec/hatchet/python_spec.rb

@@ -4,7 +4,7 @@ describe "Python!!!!!!!!!!!" do
   it "🐍" do
     Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app|
       expect(app.output).to           match(/Installing pip/)
-      expect(app.run('python -V')).to match(/3.6.8/)
+      expect(app.run('python -V')).to match(/3.6.9/)
     end
   end
 end

test/run

@@ -1,318 +0,0 @@
-#!/usr/bin/env bash
-
-# Default Python Versions
-# shellcheck source=bin/default_pythons
-source "bin/default_pythons"
-
-testAirflow() {
-  export SLUGIFY_USES_TEXT_UNIDECODE="yes"
-  compile "airflow"
-  assertCaptured "apache-airflow==1.10.2"
-  assertCapturedSuccess
-}
-
-testPipenv() {
-  compile "pipenv"
-  assertCapturedSuccess
-}
-
-testPipenvLock() {
-  compile "pipenv-lock"
-  assertCapturedSuccess
-}
-
-testPipenvVersion() {
-  compile "pipenv-version"
-  assertCaptured $DEFAULT_PYTHON_VERSION
-  assertCapturedSuccess
-}
-
-testPipenvVersion2() {
-  compile "pipenv-version2"
-  assertCaptured $LATEST_27
-  assertCapturedSuccess
-}
-testPipenvFullVersion() {
-  compile "pipenv-full-version"
-  assertCaptured "3.6.3"
-  assertCapturedSuccess
-}
-
-testNoRequirements() {
-  compile "no-requirements"
-  assertCapturedError
-}
-
-testCollectstatic() {
-  compile "collectstatic"
-  assertCaptured "collectstatic"
-}
-
-testGEOS() {
-  export BUILD_WITH_GEO_LIBRARIES=1
-  compile "geos"
-  assertCaptured "geos"
-  assertCapturedSuccess
-}
-
-testNLTK() {
-  # NOTE: This is a RuntimeWarning emitted by Python 3's runpy.py script
-  # which is what is used when you call `python -m <module>`. This is due to
-  # how nltk imports things. It's not actually an error, but it would probably
-  # be bad to silence in Production.
-  export PYTHONWARNINGS="ignore::RuntimeWarning"
-  compile "nltk"
-  assertCaptured "[nltk_data] Downloading package city_database" "STD_ERR"
-  assertCapturedSuccess
-}
-
-testSetupPy() {
-  compile "setup-py"
-  assertCaptured "maya"
-  assertCapturedSuccess
-}
-
-
-testStandardRequirements() {
-  compile "requirements-standard"
-  assertCaptured "requests"
-  assertCapturedSuccess
-}
-
-testPsycopg2() {
-  compile "psycopg2"
-  assertCaptured "psycopg2"
-  assertCapturedSuccess
-}
-
-testCffi() {
-  compile "cffi"
-  assertCaptured "cffi"
-  assertCapturedSuccess
-}
-
-testPylibmc() {
-  compile "pylibmc"
-  assertCaptured "pylibmc"
-  assertCapturedSuccess
-}
-
-testPythonDefault() {
-  updateVersion "pythonDefault" $DEFAULT_PYTHON_VERSION
-  compile "pythonDefault"
-  assertCaptured $DEFAULT_PYTHON_VERSION
-  assertNotCaptured "security update"
-  assertCapturedSuccess
-}
-
-testPython2() {
-    updateVersion "python2" $LATEST_27
-    echo $LATEST_27 > "runtime.txt"
-    compile "python2"
-    assertCaptured $LATEST_27
-    assertNotCaptured "security update"
-    assertCapturedSuccess
-}
-
-testPython2_warn() {
-    compile "python2_warn"
-    assertCaptured "python-2.7.15"
-    assertCaptured "security update!"
-    assertCapturedSuccess
-}
-
-testPython2_fail() {
-    compile "python2_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
-}
-
-testPython3_4() {
-  if [[ $STACK != "cedar-14" ]]; then
-    updateVersion "python3_4" $LATEST_34
-    compile "python3_4"
-    assertCaptured $LATEST_34
-    assertNotCaptured "security update"
-    assertCapturedSuccess
-  fi
-}
-
-testPython3_4_warn() {
-  compile "python3_4_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCaptured "python-3.4.0"
-    assertCaptured "security update!"
-    assertCapturedSuccess
-  else
-    assertCapturedError
-  fi
-}
-
-testPython3_4_fail() {
-    compile "python3_4_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
-}
-
-testPython3_5() {
-  if [[ $STACK != "cedar-14" ]]; then
-    updateVersion "python3_5" $LATEST_35
-    compile "python3_5"
-    assertCaptured $LATEST_35
-    assertNotCaptured "security update"
-    assertCapturedSuccess
-  fi
-}
-
-testPython3_5_warn() {
-  compile "python3_5_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCaptured "python-3.5.3"
-    assertCaptured "security update!"
-    assertCapturedError
-  else
-    assertCapturedError
-  fi
-}
-
-testPython3_5_fail() {
-    compile "python3_5_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
-}
-
-testPython3_6() {
-  updateVersion "python3_6" $LATEST_36
-  compile "python3_6"
-  assertCaptured $LATEST_36
-  assertNotCaptured "security update"
-  assertCapturedSuccess
-}
-
-testPython3_6_warn() {
-  compile "python3_6_warn"
-  assertCaptured "python-3.6.7"
-  assertCaptured "security update!"
-  assertCapturedSuccess
-}
-
-testPython3_6_fail() {
-    compile "python3_6_fail"
-    assertCaptured "Aborting"
-    assertCapturedError
-}
-
-testPython3_7() {
-  updateVersion "python3_7" $LATEST_37
-  compile "python3_7"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertNotCaptured "security update"
-    assertCaptured $LATEST_37
-    assertCapturedSuccess
-  fi
-}
-
-testPython3_7_warn() {
-  compile "python3_7_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "python-3.7.1"
-    assertCaptured "security update!"
-    assertCapturedSuccess
-  fi
-}
-
-testPython3_7_fail() {
-  compile "python3_7_fail"
-  assertCaptured "Aborting"
-  assertCapturedError
-}
-
-testGitEgg() {
-  compile "git-egg"
-  assertCaptured "requests"
-  assertCapturedSuccess
-}
-
-testSmartRequirements() {
-  local cache_dir="$(mktmpdir)"
-  compile "requirements-standard" "$cache_dir"
-  assertFile "requests" ".heroku/python/requirements-declared.txt"
-  assertCapturedSuccess
-  compile "psycopg2" "$cache_dir"
-  assertCaptured "Uninstalling requests"
-  assertFile "psycopg2" ".heroku/python/requirements-declared.txt"
-  assertCapturedSuccess
-}
-
-testStackChange() {
-  local cache_dir="$(mktmpdir)"
-  mkdir -p "${cache_dir}/.heroku"
-  echo "different-stack" > "${cache_dir}/.heroku/python-stack"
-  compile "requirements-standard" "$cache_dir"
-  assertCaptured "clearing cache"
-  assertFile "$STACK" ".heroku/python-stack"
-  assertCapturedSuccess
-}
-
-
-pushd $(dirname 0) >/dev/null
-popd >/dev/null
-
-source $(pwd)/test/utils
-
-mktmpdir() {
-  dir=$(mktemp -t testXXXXX)
-  rm -rf $dir
-  mkdir $dir
-  echo $dir
-}
-
-detect() {
-  capture $(pwd)/bin/detect $(pwd)/test/fixtures/$1
-}
-
-compile_dir=""
-
-default_process_types_cleanup() {
-  file="/tmp/default_process_types"
-  if [ -f "$file" ]; then
-    rm "$file"
-  fi
-}
-
-compile() {
-  default_process_types_cleanup
-  bp_dir=$(mktmpdir)
-  compile_dir=$(mktmpdir)
-  cp -a $(pwd)/* ${bp_dir}
-  cp -a ${bp_dir}/test/fixtures/$1/. ${compile_dir}
-  capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3
-}
-
-compileDir() {
-  default_process_types_cleanup
-
-  local bp_dir=$(mktmpdir)
-  local compile_dir=${1:-$(mktmpdir)}
-  local cache_dir=${2:-$(mktmpdir)}
-  local env_dir=$3
-
-  cp -a $(pwd)/* ${bp_dir}
-  capture ${bp_dir}/bin/compile ${compile_dir} ${cache_dir} ${env_dir}
-}
-
-release() {
-  bp_dir=$(mktmpdir)
-  cp -a $(pwd)/* ${bp_dir}
-  capture ${bp_dir}/bin/release ${bp_dir}/test/fixtures/$1
-}
-
-assertFile() {
-  assertEquals "$1" "$(cat ${compile_dir}/$2)"
-}
-
-source $(pwd)/test/shunit2

test/run-deps

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+# Default Python Versions
+# shellcheck source=bin/default_pythons
+source "bin/default_pythons"
+
+testAirflow() {
+  export SLUGIFY_USES_TEXT_UNIDECODE="yes"
+  compile "airflow"
+  assertCaptured "apache-airflow==1.10.2"
+  assertCapturedSuccess
+}
+
+testCollectstatic() {
+  compile "collectstatic"
+  assertCaptured "collectstatic"
+}
+
+testGEOS() {
+  export BUILD_WITH_GEO_LIBRARIES=1
+  compile "geos"
+  assertCaptured "geos"
+  assertCapturedSuccess
+}
+
+testNLTK() {
+  # NOTE: This is a RuntimeWarning emitted by Python 3's runpy.py script
+  # which is what is used when you call `python -m <module>`. This is due to
+  # how nltk imports things. It's not actually an error, but it would probably
+  # be bad to silence in Production.
+  export PYTHONWARNINGS="ignore::RuntimeWarning"
+  compile "nltk"
+  assertCaptured "[nltk_data] Downloading package city_database" "STD_ERR"
+  assertCapturedSuccess
+}
+
+testPsycopg2() {
+  compile "psycopg2"
+  assertCaptured "psycopg2"
+  assertCapturedSuccess
+}
+
+testCffi() {
+  compile "cffi"
+  assertCaptured "cffi"
+  assertCapturedSuccess
+}
+
+testPylibmc() {
+  compile "pylibmc"
+  assertCaptured "pylibmc"
+  assertCapturedSuccess
+}
+
+pushd $(dirname 0) >/dev/null
+popd >/dev/null
+
+source $(pwd)/test/utils
+source $(pwd)/test/shunit2

test/run-features

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+# Default Python Versions
+# shellcheck source=bin/default_pythons
+source "bin/default_pythons"
+
+testGitEgg() {
+  compile "git-egg"
+  assertCaptured "requests"
+  assertCapturedSuccess
+}
+
+testSmartRequirements() {
+  local cache_dir="$(mktmpdir)"
+  compile "requirements-standard" "$cache_dir"
+  assertFile "requests" ".heroku/python/requirements-declared.txt"
+  assertCapturedSuccess
+  compile "psycopg2" "$cache_dir"
+  assertCaptured "Uninstalling requests"
+  assertFile "psycopg2" ".heroku/python/requirements-declared.txt"
+  assertCapturedSuccess
+}
+
+testStackChange() {
+  local cache_dir="$(mktmpdir)"
+  mkdir -p "${cache_dir}/.heroku"
+  echo "different-stack" > "${cache_dir}/.heroku/python-stack"
+  compile "requirements-standard" "$cache_dir"
+  assertCaptured "clearing cache"
+  assertFile "$STACK" ".heroku/python-stack"
+  assertCapturedSuccess
+}
+
+testSetupPy() {
+  compile "setup-py"
+  assertCaptured "maya"
+  assertCapturedSuccess
+}
+
+testStandardRequirements() {
+  compile "requirements-standard"
+  assertCaptured "requests"
+  assertCapturedSuccess
+}
+
+testPipenv() {
+  compile "pipenv"
+  assertCapturedSuccess
+}
+
+testPipenvLock() {
+  compile "pipenv-lock"
+  assertCapturedSuccess
+}
+
+testPipenvVersion() {
+  compile "pipenv-version"
+  assertCaptured $DEFAULT_PYTHON_VERSION
+  assertCapturedSuccess
+}
+
+testPipenvVersion2() {
+  compile "pipenv-version2"
+  assertCaptured $LATEST_27
+  assertCapturedSuccess
+}
+testPipenvFullVersion() {
+  compile "pipenv-full-version"
+  assertCaptured "3.6.3"
+  assertCapturedSuccess
+}
+
+testNoRequirements() {
+  compile "no-requirements"
+  assertCapturedError
+}
+
+pushd $(dirname 0) >/dev/null
+popd >/dev/null
+
+source $(pwd)/test/utils
+source $(pwd)/test/shunit2

test/run-versions

@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+
+# Default Python Versions
+# shellcheck source=bin/default_pythons
+source "bin/default_pythons"
+
+testPythonDefault() {
+  updateVersion "pythonDefault" $DEFAULT_PYTHON_VERSION
+  compile "pythonDefault"
+  assertCaptured $DEFAULT_PYTHON_VERSION
+  assertNotCaptured "security update"
+  assertCapturedSuccess
+}
+
+testPython2() {
+    updateVersion "python2" $LATEST_27
+    echo $LATEST_27 > "runtime.txt"
+    compile "python2"
+    assertCaptured $LATEST_27
+    assertNotCaptured "security update"
+    assertCapturedSuccess
+}
+
+testPython2_warn() {
+    compile "python2_warn"
+    assertCaptured "python-2.7.15"
+    assertCaptured "security update!"
+    assertCapturedSuccess
+}
+
+testPython2_fail() {
+    compile "python2_fail"
+    assertCaptured "Aborting"
+    assertCapturedError
+}
+
+testPython3_4() {
+  if [[ $STACK != "cedar-14" ]]; then
+    updateVersion "python3_4" $LATEST_34
+    compile "python3_4"
+    assertCaptured $LATEST_34
+    assertNotCaptured "security update"
+    assertCapturedSuccess
+  fi
+}
+
+testPython3_4_warn() {
+  compile "python3_4_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCaptured "python-3.4.0"
+    assertCaptured "security update!"
+    assertCapturedSuccess
+  else
+    assertCapturedError
+  fi
+}
+
+testPython3_4_fail() {
+    compile "python3_4_fail"
+    assertCaptured "Aborting"
+    assertCapturedError
+}
+
+testPython3_5() {
+  if [[ $STACK != "cedar-14" ]]; then
+    updateVersion "python3_5" $LATEST_35
+    compile "python3_5"
+    assertCaptured $LATEST_35
+    assertNotCaptured "security update"
+    assertCapturedSuccess
+  fi
+}
+
+testPython3_5_warn() {
+  compile "python3_5_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCaptured "python-3.5.3"
+    assertCaptured "security update!"
+    assertCapturedError
+  else
+    assertCapturedError
+  fi
+}
+
+testPython3_5_fail() {
+    compile "python3_5_fail"
+    assertCaptured "Aborting"
+    assertCapturedError
+}
+
+testPython3_6() {
+  updateVersion "python3_6" $LATEST_36
+  compile "python3_6"
+  assertCaptured $LATEST_36
+  assertNotCaptured "security update"
+  assertCapturedSuccess
+}
+
+testPython3_6_warn() {
+  compile "python3_6_warn"
+  assertCaptured "python-3.6.7"
+  assertCaptured "security update!"
+  assertCapturedSuccess
+}
+
+testPython3_6_fail() {
+    compile "python3_6_fail"
+    assertCaptured "Aborting"
+    assertCapturedError
+}
+
+testPython3_7() {
+  updateVersion "python3_7" $LATEST_37
+  compile "python3_7"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertNotCaptured "security update"
+    assertCaptured $LATEST_37
+    assertCapturedSuccess
+  fi
+}
+
+testPython3_7_warn() {
+  compile "python3_7_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "python-3.7.1"
+    assertCaptured "security update!"
+    assertCapturedSuccess
+  fi
+}
+
+testPython3_7_fail() {
+  compile "python3_7_fail"
+  assertCaptured "Aborting"
+  assertCapturedError
+}
+
+pushd $(dirname 0) >/dev/null
+popd >/dev/null
+
+source $(pwd)/test/utils
+source $(pwd)/test/shunit2

test/utils

@@ -212,3 +212,55 @@ assertFileMD5()
 
   assertEquals "${expected_md5_cmd_output}" "`${md5_cmd}`"
 }
+
+# Test helpers
+mktmpdir() {
+  dir=$(mktemp -t testXXXXX)
+  rm -rf $dir
+  mkdir $dir
+  echo $dir
+}
+
+detect() {
+  capture $(pwd)/bin/detect $(pwd)/test/fixtures/$1
+}
+
+compile_dir=""
+
+default_process_types_cleanup() {
+  file="/tmp/default_process_types"
+  if [ -f "$file" ]; then
+    rm "$file"
+  fi
+}
+
+compile() {
+  default_process_types_cleanup
+  bp_dir=$(mktmpdir)
+  compile_dir=$(mktmpdir)
+  cp -a $(pwd)/* ${bp_dir}
+  cp -a ${bp_dir}/test/fixtures/$1/. ${compile_dir}
+  capture ${bp_dir}/bin/compile ${compile_dir} ${2:-$(mktmpdir)} $3
+}
+
+compileDir() {
+  default_process_types_cleanup
+
+  local bp_dir=$(mktmpdir)
+  local compile_dir=${1:-$(mktmpdir)}
+  local cache_dir=${2:-$(mktmpdir)}
+  local env_dir=$3
+
+  cp -a $(pwd)/* ${bp_dir}
+  capture ${bp_dir}/bin/compile ${compile_dir} ${cache_dir} ${env_dir}
+}
+
+release() {
+  bp_dir=$(mktmpdir)
+  cp -a $(pwd)/* ${bp_dir}
+  capture ${bp_dir}/bin/release ${bp_dir}/test/fixtures/$1
+}
+
+assertFile() {
+  assertEquals "$1" "$(cat ${compile_dir}/$2)"
+}