Next version (#405)

* python 3.5.3

* v108

* new setuptools version

* removed unneeded variable

.dockerignore

@@ -0,0 +1 @@
+.git/

CHANGELOG.md

@@ -1,5 +1,45 @@
 # Python Buildpack Changelog
 
+# 108
+
+Updated setuptools.
+
+# 107
+
+Bugfix for C dependency installation.
+
+# 106
+
+ Don't install packages that could mess up packaging.
+
+ - The Python buildpack will automatically remove `six`, `pyparsing`, `appdirs`,
+   `setuptools`, and `distribute` from a `requirements.txt` file now, as these
+   packages are provided by the Python buildpack.
+
+ # 105
+
+ Improvements to output messaging.
+
+# 104
+
+General improvements.
+
+- Fix for Heroku CI.
+- Use `pkg_resources` to check if a distribution is installed instead of
+  parsing `requirements.txt`. ([#395][395])
+
+[395]: https://github.com/heroku/heroku-buildpack-python/pull/395
+
+## 103
+
+Bug fixes and improvements.
+
+- Fix for Pipenv.
+- Fix for Heroku CI.
+- Improve handling of WEB_CONCURRENCY when using multiple buildpacks.
+- Adjust environment variables set during the build to more closely match those in the dyno environment (DYNO is now available, STACK is not).
+- Restore the build cache prior to running bin/pre_compile.
+
 ## 102
 
 Buildpack code cleanup.

Dockerfile

@@ -1,11 +1,13 @@
 FROM heroku/heroku:16-build
 
 WORKDIR /app
-ENV WORKSPACE_DIR=/app/builds
+ENV WORKSPACE_DIR="/app/builds" \
+    S3_BUCKET="lang-python" \
+    S3_PREFIX="heroku-16/"
 
-RUN apt-get update && apt-get install -y python-pip
+RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
 
-# Install bob-builder application
-RUN pip install bob-builder==0.0.5
+COPY requirements.txt /app/
+RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
 
-COPY . /app
+COPY . /app

Makefile

@@ -13,7 +13,20 @@ test-heroku-16:
 	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
 	@echo ""
 
+buildenv-heroku-16:
+	@echo "Creating build environment (heroku-16)..."
+	@echo
+	@docker build --pull -t python-buildenv-heroku-16 .
+	@echo
+	@echo "Usage..."
+	@echo
+	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
+	@echo "  $$ bob build runtimes/python-2.7.13"
+	@echo "  $$ bob deploy runtimes/python-2.7.13"
+	@echo
+	@docker run -it --rm python-buildenv-heroku-16
+
 tools:
 	git clone https://github.com/kennethreitz/pip-pop.git
 	mv pip-pop/bin/* vendor/pip-pop/
-	rm -fr pip-pop
+	rm -fr pip-pop

README.md

@@ -52,5 +52,5 @@ Runtime options include:
 
 - `python-2.7.13`
 - `python-3.6.1`
-- `pypy-5.7.0` (unsupported, experimental)
-- `pypy3-5.5.0` (unsupported, experimental)
+- `pypy-5.7.1` (unsupported, experimental)
+- `pypy3-5.5.1` (unsupported, experimental)

bin/compile

@@ -36,7 +36,6 @@ DEFAULT_PYTHON_VERSION="python-2.7.13"
 DEFAULT_PYTHON_STACK="cedar-14"
 PYTHON_EXE="/app/.heroku/python/bin/python"
 PIP_VERSION="9.0.1"
-SETUPTOOLS_VERSION="32.1.0"
 
 # Common Problem Warnings
 export WARNINGS_LOG=$(mktemp)
@@ -68,6 +67,7 @@ mkdir -p /app/.heroku
 PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
 EXPORT_PATH="$BIN_DIR/../export"
 GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh"
+WEB_CONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/WEB_CONCURRENCY.sh"
 
 # We'll need to send these statics to other scripts we `source`.
 export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH
@@ -91,6 +91,20 @@ if [[ ! -f Procfile ]]; then
   puts-warn 'Learn more: https://devcenter.heroku.com/articles/procfile'
 fi
 
+# Prepare the cache.
+mkdir -p $CACHE_DIR
+
+# Restore old artifacts from the cache.
+mkdir -p .heroku
+
+cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
+if [[ -d $CACHE_DIR/.heroku/src ]]; then
+  cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
+fi
+
 # Experimental pre_compile hook.
 source $BIN_DIR/steps/hooks/pre_compile
 
@@ -114,21 +128,6 @@ if [ ! -f runtime.txt ]; then
   echo $DEFAULT_PYTHON_VERSION > runtime.txt
 fi
 
-# Prepare the cache.
-mkdir -p $CACHE_DIR
-
-# Restore old artifacts from the cache.
-mkdir -p .heroku
-
-cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
-if [[ -d $CACHE_DIR/.heroku/src ]]; then
-  cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
-fi
-
-
 mkdir -p $(dirname $PROFILE_PATH)
 mkdir -p /app/.heroku/src
 
@@ -145,11 +144,6 @@ let start=$(nowms)
 source $BIN_DIR/steps/python
 mtime "python.install.time" "${start}"
 
-# Sanity check for setuptools/distribute.
-let start=$(nowms)
-source $BIN_DIR/steps/setuptools
-mtime "setuptools.install.time" "${start}"
-
 # Pipenv support.
 source $BIN_DIR/steps/pipenv
 
@@ -192,8 +186,12 @@ sub-env $BIN_DIR/steps/nltk
 mtime "nltk.download.time" "${start}"
 
 # Support for pip install -e.
-rm -fr $BUILD_DIR/.heroku/src
-deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
+# In CI, $BUILD_DIR is /app.
+if [[ ! "$BUILD_DIR" == "/app" ]]; then
+  rm -fr $BUILD_DIR/.heroku/src
+  deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
+fi
+
 
 # Django collectstatic support.
 let start=$(nowms)
@@ -211,8 +209,10 @@ set-default-env PYTHONHASHSEED random
 set-default-env PYTHONPATH /app/
 
 # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.
+cp $ROOT_DIR/vendor/WEB_CONCURRENCY.sh $WEB_CONCURRENCY_PROFILE_PATH
 cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH
 
+
 # Experimental post_compile hook.
 source $BIN_DIR/steps/hooks/post_compile
 

bin/steps/collectstatic

@@ -20,7 +20,7 @@ MANAGE_FILE=${MANAGE_FILE:-fakepath}
 [ -f .heroku/collectstatic_disabled ] && DISABLE_COLLECTSTATIC=1
 
 # Ensure that Django is explicitly specified in requirements.txt
-pip-grep -s requirements.txt django Django && DJANGO_INSTALLED=1
+sp-grep django && DJANGO_INSTALLED=1
 
 
 if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then

bin/steps/pip-install

@@ -1,20 +1,23 @@
-# Install dependencies with Pip.
-puts-step "Installing requirements with pip"
+if [ ! "$SKIP_PIP_INSTALL" ]; then
 
-set +e
-/app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
-PIP_STATUS="${PIPESTATUS[0]}"
-set -e
+    # Install dependencies with Pip.
+    puts-step "Installing requirements with pip"
 
-show-warnings
+    set +e
+    /app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
+    PIP_STATUS="${PIPESTATUS[0]}"
+    set -e
 
-if [[ ! $PIP_STATUS -eq 0 ]]; then
-    exit 1
-fi
+    show-warnings
+
+    if [[ ! $PIP_STATUS -eq 0 ]]; then
+        exit 1
+    fi
 
 
-# Smart Requirements handling
-cp requirements.txt .heroku/python/requirements-declared.txt
-/app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
+    # Smart Requirements handling
+    cp requirements.txt .heroku/python/requirements-declared.txt
+    /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
 
-echo
+    echo
+fi

bin/steps/pipenv

@@ -1,10 +1,19 @@
 # Pipenv support (Generate requriements.txt with pipenv).
 if [[ -f Pipfile ]]; then
     if [[ ! -f requirements.txt ]]; then
-        puts-step "Generating 'requirements.txt' with pipenv"
+        puts-step "Installing requirements with latest pipenv..."
 
+        # Install pipenv.
         /app/.heroku/python/bin/pip install pipenv --upgrade &> /dev/null
 
-        /app/.heroku/python/bin/pipenv lock --requirements --no-hashes > $BUILD_DIR/requirements.txt 2> /dev/null
+        # Install the dependencies.
+        /app/.heroku/python/bin/pipenv install --system 2>&1 | indent
+
+        # Skip pip install, later.
+        export SKIP_PIP_INSTALL=1
+
+        # Pip freeze, for compatibility.
+        /app/.heroku/python/bin/pip freeze > requirements.txt
+
     fi
 fi

bin/steps/setuptools

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-# Syntax sugar.
-source $BIN_DIR/utils
-
-if (pip-grep -s requirements.txt setuptools distribute &> /dev/null) then
-
-  puts-warn 'The package setuptools/distribute is listed in requirements.txt.'
-  puts-warn 'Please remove to ensure expected behavior. '
-
-fi

bin/utils

@@ -3,7 +3,7 @@ shopt -s extglob
 
 # The standard library.
 if [[ ! -f  /tmp/stdlib.sh ]]; then
-  curl --retry 3 -s https://raw.githubusercontent.com/heroku/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
+  curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
 fi
 source /tmp/stdlib.sh
 

builds/README.md

@@ -1,5 +1,7 @@
 # Python Buildpack Binaries
 
+For Cedar-14 stack
+------------------
 
 To get started with it, create an app on Heroku inside a clone of this repository, and set your S3 config vars:
 
@@ -28,4 +30,12 @@ If this works, run `bob deploy` instead of `bob build` to have the result upload
 
 To speed things up drastically, it'll usually be a good idea to `heroku run bash --size PX` instead.
 
-Enjoy :)
+For Heroku-16 stack
+-------------------
+
+1. Ensure GNU Make and Docker are installed.
+2. From the root of the buildpack repository, run: `make buildenv-heroku-16`
+3. Follow the instructions displayed!
+
+
+Enjoy :)

builds/runtimes/pypy-5.7.1

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+
+OUT_PREFIX=$1
+
+echo "Building PyPy..."
+SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux64.tar.bz2'
+curl -L $SOURCE_TARBALL | tar jx
+cp -R pypy2-v5.7.1-linux64/* $OUT_PREFIX
+
+ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/pypy3-5.7.1

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building PyPy..."
+SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-linux64.tar.bz2'
+curl -L $SOURCE_TARBALL | tar jx
+cp -R pypy3-v5.7.1-linux64/* $OUT_PREFIX
+
+ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python

builds/runtimes/python-3.5.3

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python..."
+SOURCE_TARBALL='https://python.org/ftp/python/3.5.3/Python-3.5.3.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.5.3 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python
+

requirements.txt

@@ -1 +1 @@
-bob-builder==0.0.5
+bob-builder==0.0.13

test/utils

@@ -93,8 +93,14 @@ assertNotCaptured()
 
 assertCapturedSuccess()
 {
-  assertEquals "Expected captured exit code to be 0; was <${RETURN}>" "0" "${RETURN}"
-  assertEquals "Expected STD_ERR to be empty; was <$(cat ${STD_ERR})>" "" "$(cat ${STD_ERR})"
+  assertEquals "Captured exit code -" "0" "${RETURN}"
+  assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
+
+  if [ $RETURN -ne 0 -a -z "$(cat ${STD_ERR})" ]; then
+    # Failing exit code but stderr was empty. Display stdout to help debugging.
+    cat $STD_OUT
+    echo
+  fi
 }
 
 # assertCapturedError [[expectedErrorCode] expectedErrorMsg]

vendor/WEB_CONCURRENCY.sh

@@ -0,0 +1,29 @@
+case $(ulimit -u) in
+
+# Automatic configuration for Gunicorn's Workers setting.
+
+# Standard-1X (+Free, +Hobby) Dyno
+256)
+  export DYNO_RAM=512
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-02}
+  ;;
+
+# Standard-2X Dyno
+512)
+  export DYNO_RAM=1024
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-04}
+  ;;
+
+# Performance-M Dyno
+16384)
+  export DYNO_RAM=2560
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-08}
+  ;;
+
+# Performance-L Dyno
+32768)
+  export DYNO_RAM=6656
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-011}
+  ;;
+
+esac

vendor/pip-pop/sp-grep

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Usage:
+  sp-grep [-s] <package>...
+
+Options:
+  -h --help     Show this screen.
+"""
+from docopt import docopt
+from pkg_resources import DistributionNotFound, get_distribution
+
+
+def has_any_distribution(names, silent=False):
+    for name in names:
+        try:
+            get_distribution(name)
+        except DistributionNotFound:
+            continue
+
+        if not silent:
+            print('Package {name} found!'.format(name=name))
+
+        exit(0)
+
+    if not silent:
+        print('Not found.')
+
+    exit(1)
+
+
+def main():
+    args = docopt(__doc__, version='sp-grep')
+    has_any_distribution(names=args['<package>'], silent=args['-s'])
+
+
+if __name__ == '__main__':
+    main()

vendor/python.gunicorn.sh

@@ -1,38 +1,2 @@
-if [[ "${WEB_CONCURRENCY:-}" == 0* ]]; then
-  # another buildpack set a default value, with leading zero
-  unset WEB_CONCURRENCY
-fi
-
-case $(ulimit -u) in
-
-# Automatic configuration for Gunicorn's Workers setting.
-# Leading zero padding so a subsequent buildpack can figure out that we set a value, and not the user
-
-# Standard-1X (+Free, +Hobby) Dyno
-256)
-  export DYNO_RAM=512
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-02}
-  ;;
-
-# Standard-2X Dyno
-512)
-  export DYNO_RAM=1024
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-04}
-  ;;
-
-# Performance-M Dyno
-16384)
-  export DYNO_RAM=2560
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-08}
-  ;;
-
-# Performance-L Dyno
-32768)
-  export DYNO_RAM=6656
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-011}
-  ;;
-
-esac
-
 # Automatic configuration for Gunicorn's ForwardedAllowIPS setting.
 export FORWARDED_ALLOW_IPS='*'