updated changelog

more accurate than parsing requirements.txt
fixes #359

.dockerignore

@@ -0,0 +1 @@
+.git/

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Python Buildpack Changelog
 
+# 104
+
+General improvements.
+
+- Fix for Heroku CI.
+- Use `pkg_resources` to check if a distribution is installed instead of
+  parsing `requirements.txt`. ([#395][395])
+
+[395]: https://github.com/heroku/heroku-buildpack-python/pull/395
+
+## 103
+
+Bug fixes and improvements.
+
+- Fix for Pipenv.
+- Fix for Heroku CI.
+- Improve handling of WEB_CONCURRENCY when using multiple buildpacks.
+- Adjust environment variables set during the build to more closely match those in the dyno environment (DYNO is now available, STACK is not).
+- Restore the build cache prior to running bin/pre_compile.
+
 ## 102
 
 Buildpack code cleanup.

Dockerfile

@@ -1,11 +1,13 @@
 FROM heroku/heroku:16-build
 
 WORKDIR /app
-ENV WORKSPACE_DIR=/app/builds
+ENV WORKSPACE_DIR="/app/builds" \
+    S3_BUCKET="lang-python" \
+    S3_PREFIX="heroku-16/"
 
-RUN apt-get update && apt-get install -y python-pip
+RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
 
-# Install bob-builder application
-RUN pip install bob-builder==0.0.5
+COPY requirements.txt /app/
+RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
 
-COPY . /app
+COPY . /app

Makefile

@@ -13,7 +13,20 @@ test-heroku-16:
 	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
 	@echo ""
 
+buildenv-heroku-16:
+	@echo "Creating build environment (heroku-16)..."
+	@echo
+	@docker build --pull -t python-buildenv-heroku-16 .
+	@echo
+	@echo "Usage..."
+	@echo
+	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
+	@echo "  $$ bob build runtimes/python-2.7.13"
+	@echo "  $$ bob deploy runtimes/python-2.7.13"
+	@echo
+	@docker run -it --rm python-buildenv-heroku-16
+
 tools:
 	git clone https://github.com/kennethreitz/pip-pop.git
 	mv pip-pop/bin/* vendor/pip-pop/
-	rm -fr pip-pop
+	rm -fr pip-pop

README.md

@@ -52,5 +52,5 @@ Runtime options include:
 
 - `python-2.7.13`
 - `python-3.6.1`
-- `pypy-5.7.0` (unsupported, experimental)
-- `pypy3-5.5.0` (unsupported, experimental)
+- `pypy-5.7.1` (unsupported, experimental)
+- `pypy3-5.5.1` (unsupported, experimental)

bin/compile

@@ -68,6 +68,7 @@ mkdir -p /app/.heroku
 PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
 EXPORT_PATH="$BIN_DIR/../export"
 GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh"
+WEB_CONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/WEB_CONCURRENCY.sh"
 
 # We'll need to send these statics to other scripts we `source`.
 export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH
@@ -91,6 +92,20 @@ if [[ ! -f Procfile ]]; then
   puts-warn 'Learn more: https://devcenter.heroku.com/articles/procfile'
 fi
 
+# Prepare the cache.
+mkdir -p $CACHE_DIR
+
+# Restore old artifacts from the cache.
+mkdir -p .heroku
+
+cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
+if [[ -d $CACHE_DIR/.heroku/src ]]; then
+  cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
+fi
+
 # Experimental pre_compile hook.
 source $BIN_DIR/steps/hooks/pre_compile
 
@@ -114,21 +129,6 @@ if [ ! -f runtime.txt ]; then
   echo $DEFAULT_PYTHON_VERSION > runtime.txt
 fi
 
-# Prepare the cache.
-mkdir -p $CACHE_DIR
-
-# Restore old artifacts from the cache.
-mkdir -p .heroku
-
-cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
-if [[ -d $CACHE_DIR/.heroku/src ]]; then
-  cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
-fi
-
-
 mkdir -p $(dirname $PROFILE_PATH)
 mkdir -p /app/.heroku/src
 
@@ -192,8 +192,12 @@ sub-env $BIN_DIR/steps/nltk
 mtime "nltk.download.time" "${start}"
 
 # Support for pip install -e.
-rm -fr $BUILD_DIR/.heroku/src
-deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
+# In CI, $BUILD_DIR is /app.
+if [[ ! "$BUILD_DIR" == "/app" ]]; then
+  rm -fr $BUILD_DIR/.heroku/src
+  deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
+fi
+
 
 # Django collectstatic support.
 let start=$(nowms)
@@ -211,8 +215,10 @@ set-default-env PYTHONHASHSEED random
 set-default-env PYTHONPATH /app/
 
 # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.
+cp $ROOT_DIR/vendor/WEB_CONCURRENCY.sh $WEB_CONCURRENCY_PROFILE_PATH
 cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH
 
+
 # Experimental post_compile hook.
 source $BIN_DIR/steps/hooks/post_compile
 

bin/steps/collectstatic

@@ -20,7 +20,7 @@ MANAGE_FILE=${MANAGE_FILE:-fakepath}
 [ -f .heroku/collectstatic_disabled ] && DISABLE_COLLECTSTATIC=1
 
 # Ensure that Django is explicitly specified in requirements.txt
-pip-grep -s requirements.txt django Django && DJANGO_INSTALLED=1
+pip-grep -s Django && DJANGO_INSTALLED=1
 
 
 if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then

bin/steps/cryptography

@@ -18,7 +18,7 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 source $BIN_DIR/utils
 
 # If a package using cffi exists within requirements, use vendored libffi.
-if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[argon2] Django[argon2] django[bcrypt] Django[bcrypt] PyNaCl pyOpenSSL PyOpenSSL requests[security] misaka &> /dev/null) then
+if (pip-grep -s argon2-cffi bcrypt cffi cryptography PyNaCl pyOpenSSL PyOpenSSL misaka &> /dev/null) then
 
   if [ ! -d ".heroku/vendor/lib/libffi-3.1" ]; then
     echo "-----> Noticed cffi. Bootstrapping libffi."

bin/steps/gdal

@@ -18,7 +18,7 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 source $BIN_DIR/utils
 
 # If GDAL exists within requirements, use vendored gdal.
-if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
+if (pip-grep -s GDAL pygdal &> /dev/null) then
 
   if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then
     echo "-----> Noticed GDAL. Bootstrapping gdal."

bin/steps/pip-install

@@ -1,20 +1,23 @@
-# Install dependencies with Pip.
-puts-step "Installing requirements with pip"
+if [ ! "$SKIP_PIP_INSTALL" ]; then
 
-set +e
-/app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
-PIP_STATUS="${PIPESTATUS[0]}"
-set -e
+    # Install dependencies with Pip.
+    puts-step "Installing requirements with pip"
 
-show-warnings
+    set +e
+    /app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
+    PIP_STATUS="${PIPESTATUS[0]}"
+    set -e
 
-if [[ ! $PIP_STATUS -eq 0 ]]; then
-    exit 1
-fi
+    show-warnings
+
+    if [[ ! $PIP_STATUS -eq 0 ]]; then
+        exit 1
+    fi
 
 
-# Smart Requirements handling
-cp requirements.txt .heroku/python/requirements-declared.txt
-/app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
+    # Smart Requirements handling
+    cp requirements.txt .heroku/python/requirements-declared.txt
+    /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
 
-echo
+    echo
+fi

bin/steps/pipenv

@@ -1,10 +1,19 @@
 # Pipenv support (Generate requriements.txt with pipenv).
 if [[ -f Pipfile ]]; then
     if [[ ! -f requirements.txt ]]; then
-        puts-step "Generating 'requirements.txt' with pipenv"
+        puts-step "Installing requirements with latest pipenv..."
 
+        # Install pipenv.
         /app/.heroku/python/bin/pip install pipenv --upgrade &> /dev/null
 
-        /app/.heroku/python/bin/pipenv lock --requirements --no-hashes > $BUILD_DIR/requirements.txt 2> /dev/null
+        # Install the dependencies.
+        /app/.heroku/python/bin/pipenv install --system 2>&1 | indent
+
+        # Skip pip install, later.
+        export SKIP_PIP_INSTALL=1
+
+        # Pip freeze, for compatibility.
+        /app/.heroku/python/bin/pip freeze > requirements.txt
+
     fi
 fi

bin/steps/pylibmc

@@ -17,7 +17,7 @@ source $BIN_DIR/utils
 
 
 # If pylibmc exists within requirements, use vendored libmemcached.
-if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
+if (pip-grep -s pylibmc &> /dev/null) then
 
   if [ ! -d ".heroku/vendor/lib/sasl2" ]; then
     echo "-----> Noticed pylibmc. Bootstrapping libmemcached."

bin/steps/setuptools

@@ -3,7 +3,7 @@
 # Syntax sugar.
 source $BIN_DIR/utils
 
-if (pip-grep -s requirements.txt setuptools distribute &> /dev/null) then
+if (pip-grep -s setuptools distribute &> /dev/null) then
 
   puts-warn 'The package setuptools/distribute is listed in requirements.txt.'
   puts-warn 'Please remove to ensure expected behavior. '

bin/utils

@@ -3,7 +3,7 @@ shopt -s extglob
 
 # The standard library.
 if [[ ! -f  /tmp/stdlib.sh ]]; then
-  curl -s https://raw.githubusercontent.com/heroku/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
+  curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
 fi
 source /tmp/stdlib.sh
 

builds/README.md

@@ -1,5 +1,7 @@
 # Python Buildpack Binaries
 
+For Cedar-14 stack
+------------------
 
 To get started with it, create an app on Heroku inside a clone of this repository, and set your S3 config vars:
 
@@ -28,4 +30,12 @@ If this works, run `bob deploy` instead of `bob build` to have the result upload
 
 To speed things up drastically, it'll usually be a good idea to `heroku run bash --size PX` instead.
 
-Enjoy :)
+For Heroku-16 stack
+-------------------
+
+1. Ensure GNU Make and Docker are installed.
+2. From the root of the buildpack repository, run: `make buildenv-heroku-16`
+3. Follow the instructions displayed!
+
+
+Enjoy :)

builds/runtimes/pypy-5.7.1

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+
+OUT_PREFIX=$1
+
+echo "Building PyPy..."
+SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux64.tar.bz2'
+curl -L $SOURCE_TARBALL | tar jx
+cp -R pypy2-v5.7.1-linux64/* $OUT_PREFIX
+
+ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/pypy3-5.7.1

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building PyPy..."
+SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-linux64.tar.bz2'
+curl -L $SOURCE_TARBALL | tar jx
+cp -R pypy3-v5.7.1-linux64/* $OUT_PREFIX
+
+ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python

requirements.txt

@@ -1 +1,2 @@
-bob-builder==0.0.5
+# TODO: Replace this once the bob-builder changes on master are released:
+https://github.com/kennethreitz/bob-builder/archive/54211376a8fb49c67ecbd6798bd45b55f4d125f4.zip

test/utils

@@ -93,8 +93,14 @@ assertNotCaptured()
 
 assertCapturedSuccess()
 {
-  assertEquals "Expected captured exit code to be 0; was <${RETURN}>" "0" "${RETURN}"
-  assertEquals "Expected STD_ERR to be empty; was <$(cat ${STD_ERR})>" "" "$(cat ${STD_ERR})"
+  assertEquals "Captured exit code -" "0" "${RETURN}"
+  assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
+
+  if [ $RETURN -ne 0 -a -z "$(cat ${STD_ERR})" ]; then
+    # Failing exit code but stderr was empty. Display stdout to help debugging.
+    cat $STD_OUT
+    echo
+  fi
 }
 
 # assertCapturedError [[expectedErrorCode] expectedErrorMsg]

vendor/WEB_CONCURRENCY.sh

@@ -0,0 +1,29 @@
+case $(ulimit -u) in
+
+# Automatic configuration for Gunicorn's Workers setting.
+
+# Standard-1X (+Free, +Hobby) Dyno
+256)
+  export DYNO_RAM=512
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-02}
+  ;;
+
+# Standard-2X Dyno
+512)
+  export DYNO_RAM=1024
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-04}
+  ;;
+
+# Performance-M Dyno
+16384)
+  export DYNO_RAM=2560
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-08}
+  ;;
+
+# Performance-L Dyno
+32768)
+  export DYNO_RAM=6656
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-011}
+  ;;
+
+esac

vendor/pip-pop/pip-grep

@@ -2,58 +2,26 @@
 # -*- coding: utf-8 -*-
 
 """Usage:
-  pip-grep [-s] <reqfile> <package>...
+  pip-grep [-s] <package>...
 
 Options:
   -h --help     Show this screen.
 """
-import os
 from docopt import docopt
-from pip.req import parse_requirements
-from pip.index import PackageFinder
-from pip._vendor.requests import session
-
-requests = session()
+from pkg_resources import DistributionNotFound, get_distribution
 
 
-class Requirements(object):
-    def __init__(self, reqfile=None):
-        super(Requirements, self).__init__()
-        self.path = reqfile
-        self.requirements = []
+def has_any_distribution(names, silent=False):
+    for name in names:
+        try:
+            get_distribution(name)
+        except DistributionNotFound:
+            continue
 
-        if reqfile:
-            self.load(reqfile)
-
-    def __repr__(self):
-        return '<Requirements \'{}\'>'.format(self.path)
-
-    def load(self, reqfile):
-        if not os.path.exists(reqfile):
-            raise ValueError('The given requirements file does not exist.')
-
-        finder = PackageFinder([], [], session=requests)
-        for requirement in parse_requirements(reqfile, finder=finder, session=requests):
-            if requirement.req:
-                if not getattr(requirement.req, 'name', None):
-                    # Prior to pip 8.1.2 the attribute `name` did not exist.
-                    requirement.req.name = requirement.req.project_name
-                self.requirements.append(requirement.req)
-
-
-def grep(reqfile, packages, silent=False):
-    try:
-        r = Requirements(reqfile)
-    except ValueError:
         if not silent:
-            print('There was a problem loading the given requirement file.')
-        exit(os.EX_NOINPUT)
+            print('Package {name} found!'.format(name=name))
 
-    for req in r.requirements:
-        if req.name in packages:
-            if not silent:
-                print('Package {} found!'.format(req.name))
-            exit(0)
+        exit(0)
 
     if not silent:
         print('Not found.')
@@ -63,10 +31,7 @@ def grep(reqfile, packages, silent=False):
 
 def main():
     args = docopt(__doc__, version='pip-grep')
-
-    kwargs = {'reqfile': args['<reqfile>'], 'packages': args['<package>'], 'silent': args['-s']}
-
-    grep(**kwargs)
+    has_any_distribution(names=args['<package>'], silent=args['-s'])
 
 
 if __name__ == '__main__':

vendor/python.gunicorn.sh

@@ -1,38 +1,2 @@
-if [[ "${WEB_CONCURRENCY:-}" == 0* ]]; then
-  # another buildpack set a default value, with leading zero
-  unset WEB_CONCURRENCY
-fi
-
-case $(ulimit -u) in
-
-# Automatic configuration for Gunicorn's Workers setting.
-# Leading zero padding so a subsequent buildpack can figure out that we set a value, and not the user
-
-# Standard-1X (+Free, +Hobby) Dyno
-256)
-  export DYNO_RAM=512
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-02}
-  ;;
-
-# Standard-2X Dyno
-512)
-  export DYNO_RAM=1024
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-04}
-  ;;
-
-# Performance-M Dyno
-16384)
-  export DYNO_RAM=2560
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-08}
-  ;;
-
-# Performance-L Dyno
-32768)
-  export DYNO_RAM=6656
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-011}
-  ;;
-
-esac
-
 # Automatic configuration for Gunicorn's ForwardedAllowIPS setting.
 export FORWARDED_ALLOW_IPS='*'