Tests: Support running only a subset of the test suites (#1021)

Previously `make test` ran all unit test suites against all stacks, which
would take up to an hour locally. This could be sped up by using one of
the stack-specific targets (such as `make test-heroku-18`), however
there was still no way to only run one of the test suites.

Now `make test` can be controlled more precisely using optional `STACK`
and `TEST_CMD` arguments, eg:

`make test STACK=heroku-16 TEST_CMD=test/versions`

Travis has now been made to use this feature, which unblocks future
Travis speedups (such as splitting the jobs up further in #1018) and
means on Travis the correct Docker image is now used (see #958).

The `tests.sh` script has been removed since it's unused after #839 and
redundant given the make targets.

Fixes #958.
Fixes #1020.

.travis.yml

@@ -11,12 +11,9 @@ before_script:
   - gem install bundler -v 1.16.2
 
 script:
-  - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
-  - docker run --rm -e "STACK=cedar-14" travis-build-cedar-14 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-  - docker run --rm -e "STACK=heroku-16" travis-build-heroku-16 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-  - docker run --rm -e "STACK=heroku-18" travis-build-heroku-18 bash $TESTFOLDER
+  - make test STACK="cedar-14" TEST_CMD="${TESTFOLDER}"
+  - make test STACK="heroku-16" TEST_CMD="${TESTFOLDER}"
+  - make test STACK="heroku-18" TEST_CMD="${TESTFOLDER}"
 
 jobs:
   include:

Makefile

@@ -1,26 +1,26 @@
 # These targets are not files
-.PHONY: tests
+.PHONY: check test buildenv-heroku-16 buildenv-heroku-18 tools
 
-test: test-heroku-18 test-heroku-16 test-cedar-14
+STACK ?= heroku-18
+TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
+
+ifeq ($(STACK),cedar-14)
+	# Cedar-14 doesn't have a build image varient.
+	IMAGE_TAG := heroku/cedar:14
+else
+	# Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
+	IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
+endif
 
 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
 	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python
 	@shellcheck -x bin/steps/hooks/*
 
-test-cedar-14:
-	@echo "Running tests in docker (cedar-14)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
+test:
+	@echo "Running tests using: STACK=$(STACK) TEST_CMD='$(TEST_CMD)'"
 	@echo ""
-
-test-heroku-16:
-	@echo "Running tests in docker (heroku-16)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
-	@echo ""
-
-test-heroku-18:
-	@echo "Running tests in docker (heroku-18)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
+	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
 	@echo ""
 
 buildenv-heroku-16:

README.md

@@ -72,17 +72,22 @@ Supported runtime options include:
 The buildpack tests use [Docker](https://www.docker.com/) to simulate
 Heroku's [stack images.](https://devcenter.heroku.com/articles/stack)
 
-To run the test suite:
+To run the test suite against the default stack:
 
 ```
 make test
 ```
 
-Or to test in a particular stack:
+Or to test against a particular stack:
 
 ```
-make test-heroku-18
-make test-heroku-16
+make test STACK=heroku-16
+```
+
+To run only a subset of the tests:
+
+```
+make test TEST_CMD=tests/versions
 ```
 
 The tests are run via the vendored

tests.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-if [[ ! "$STACK" ]]; then
-    echo '$STACK must be set! (heroku-16 | cedar-14)'
-    exit 1
-fi
-
-if [[ "$STACK" == "cedar-14" ]]; then
-    make test-cedar-14
-    exit $?
-fi
-
-if [[ "$STACK" == "heroku-16" ]]; then
-    make test-heroku-16
-    exit $?
-fi
-
-if [[ "$STACK" == "heroku-18" ]]; then
-    make test-heroku-18
-    exit $?
-fi