add retry to curl

.dockerignore

@@ -1 +0,0 @@
-.git/

CHANGELOG.md

@@ -1,37 +1,5 @@
 # Python Buildpack Changelog
 
-# 106
-
-Don't install packages that could mess up packaging.
-
-- The Python buildpack will automatically remove `six`, `pyparsing`, `appdirs`,
-  `setuptools`, and `distribute` from a `requirements.txt` file now, as these
-  packages are provided by the Python buildpack.
-
-# 105
-
-Improvements to output messaging.
-
-# 104
-
-General improvements.
-
-- Fix for Heroku CI.
-- Use `pkg_resources` to check if a distribution is installed instead of
-  parsing `requirements.txt`. ([#395][395])
-
-[395]: https://github.com/heroku/heroku-buildpack-python/pull/395
-
-## 103
-
-Bug fixes and improvements.
-
-- Fix for Pipenv.
-- Fix for Heroku CI.
-- Improve handling of WEB_CONCURRENCY when using multiple buildpacks.
-- Adjust environment variables set during the build to more closely match those in the dyno environment (DYNO is now available, STACK is not).
-- Restore the build cache prior to running bin/pre_compile.
-
 ## 102
 
 Buildpack code cleanup.

Dockerfile

@@ -1,13 +1,11 @@
 FROM heroku/heroku:16-build
 
 WORKDIR /app
-ENV WORKSPACE_DIR="/app/builds" \
-    S3_BUCKET="lang-python" \
-    S3_PREFIX="heroku-16/"
+ENV WORKSPACE_DIR=/app/builds
 
-RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y python-pip
 
-COPY requirements.txt /app/
-RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+# Install bob-builder application
+RUN pip install bob-builder==0.0.5
 
-COPY . /app
+COPY . /app

Makefile

@@ -13,20 +13,7 @@ test-heroku-16:
 	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
 	@echo ""
 
-buildenv-heroku-16:
-	@echo "Creating build environment (heroku-16)..."
-	@echo
-	@docker build --pull -t python-buildenv-heroku-16 .
-	@echo
-	@echo "Usage..."
-	@echo
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ bob build runtimes/python-2.7.13"
-	@echo "  $$ bob deploy runtimes/python-2.7.13"
-	@echo
-	@docker run -it --rm python-buildenv-heroku-16
-
 tools:
 	git clone https://github.com/kennethreitz/pip-pop.git
 	mv pip-pop/bin/* vendor/pip-pop/
-	rm -fr pip-pop
+	rm -fr pip-pop

README.md

@@ -52,5 +52,5 @@ Runtime options include:
 
 - `python-2.7.13`
 - `python-3.6.1`
-- `pypy-5.7.1` (unsupported, experimental)
-- `pypy3-5.5.1` (unsupported, experimental)
+- `pypy-5.7.0` (unsupported, experimental)
+- `pypy3-5.5.0` (unsupported, experimental)

bin/compile

@@ -68,7 +68,6 @@ mkdir -p /app/.heroku
 PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
 EXPORT_PATH="$BIN_DIR/../export"
 GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh"
-WEB_CONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/WEB_CONCURRENCY.sh"
 
 # We'll need to send these statics to other scripts we `source`.
 export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH
@@ -92,20 +91,6 @@ if [[ ! -f Procfile ]]; then
   puts-warn 'Learn more: https://devcenter.heroku.com/articles/procfile'
 fi
 
-# Prepare the cache.
-mkdir -p $CACHE_DIR
-
-# Restore old artifacts from the cache.
-mkdir -p .heroku
-
-cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
-if [[ -d $CACHE_DIR/.heroku/src ]]; then
-  cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
-fi
-
 # Experimental pre_compile hook.
 source $BIN_DIR/steps/hooks/pre_compile
 
@@ -129,6 +114,21 @@ if [ ! -f runtime.txt ]; then
   echo $DEFAULT_PYTHON_VERSION > runtime.txt
 fi
 
+# Prepare the cache.
+mkdir -p $CACHE_DIR
+
+# Restore old artifacts from the cache.
+mkdir -p .heroku
+
+cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
+if [[ -d $CACHE_DIR/.heroku/src ]]; then
+  cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
+fi
+
+
 mkdir -p $(dirname $PROFILE_PATH)
 mkdir -p /app/.heroku/src
 
@@ -145,6 +145,11 @@ let start=$(nowms)
 source $BIN_DIR/steps/python
 mtime "python.install.time" "${start}"
 
+# Sanity check for setuptools/distribute.
+let start=$(nowms)
+source $BIN_DIR/steps/setuptools
+mtime "setuptools.install.time" "${start}"
+
 # Pipenv support.
 source $BIN_DIR/steps/pipenv
 
@@ -171,19 +176,15 @@ sub-env $BIN_DIR/steps/geo-libs
 # GDAL support.
 source $BIN_DIR/steps/gdal
 
-# Uninstall removed dependencies with Pip.
-let start=$(nowms)
-source $BIN_DIR/steps/pip-uninstall
-mtime "pip.uninstall.time" "${start}"
-
-# Cleanup requirements.txt
-source $BIN_DIR/steps/setuptools
-
 # Install dependencies with Pip (where the magic happens).
 let start=$(nowms)
 source $BIN_DIR/steps/pip-install
 mtime "pip.install.time" "${start}"
 
+# Uninstall removed dependencies with Pip.
+let start=$(nowms)
+source $BIN_DIR/steps/pip-uninstall
+mtime "pip.uninstall.time" "${start}"
 
 # Support for NLTK corpora.
 let start=$(nowms)
@@ -191,12 +192,8 @@ sub-env $BIN_DIR/steps/nltk
 mtime "nltk.download.time" "${start}"
 
 # Support for pip install -e.
-# In CI, $BUILD_DIR is /app.
-if [[ ! "$BUILD_DIR" == "/app" ]]; then
-  rm -fr $BUILD_DIR/.heroku/src
-  deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
-fi
-
+rm -fr $BUILD_DIR/.heroku/src
+deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
 
 # Django collectstatic support.
 let start=$(nowms)
@@ -214,10 +211,8 @@ set-default-env PYTHONHASHSEED random
 set-default-env PYTHONPATH /app/
 
 # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.
-cp $ROOT_DIR/vendor/WEB_CONCURRENCY.sh $WEB_CONCURRENCY_PROFILE_PATH
 cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH
 
-
 # Experimental post_compile hook.
 source $BIN_DIR/steps/hooks/post_compile
 

bin/steps/collectstatic

@@ -20,7 +20,7 @@ MANAGE_FILE=${MANAGE_FILE:-fakepath}
 [ -f .heroku/collectstatic_disabled ] && DISABLE_COLLECTSTATIC=1
 
 # Ensure that Django is explicitly specified in requirements.txt
-sp-grep django && DJANGO_INSTALLED=1
+pip-grep -s requirements.txt django Django && DJANGO_INSTALLED=1
 
 
 if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then

bin/steps/pip-install

@@ -1,23 +1,20 @@
-if [ ! "$SKIP_PIP_INSTALL" ]; then
+# Install dependencies with Pip.
+puts-step "Installing requirements with pip"
 
-    # Install dependencies with Pip.
-    puts-step "Installing requirements with pip"
+set +e
+/app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
+PIP_STATUS="${PIPESTATUS[0]}"
+set -e
 
-    set +e
-    /app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
-    PIP_STATUS="${PIPESTATUS[0]}"
-    set -e
+show-warnings
 
-    show-warnings
-
-    if [[ ! $PIP_STATUS -eq 0 ]]; then
-        exit 1
-    fi
+if [[ ! $PIP_STATUS -eq 0 ]]; then
+    exit 1
+fi
 
 
-    # Smart Requirements handling
-    cp requirements.txt .heroku/python/requirements-declared.txt
-    /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
+# Smart Requirements handling
+cp requirements.txt .heroku/python/requirements-declared.txt
+/app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
 
-    echo
-fi
+echo

bin/steps/pipenv

@@ -1,19 +1,10 @@
 # Pipenv support (Generate requriements.txt with pipenv).
 if [[ -f Pipfile ]]; then
     if [[ ! -f requirements.txt ]]; then
-        puts-step "Installing requirements with latest pipenv..."
+        puts-step "Generating 'requirements.txt' with pipenv"
 
-        # Install pipenv.
         /app/.heroku/python/bin/pip install pipenv --upgrade &> /dev/null
 
-        # Install the dependencies.
-        /app/.heroku/python/bin/pipenv install --system 2>&1 | indent
-
-        # Skip pip install, later.
-        export SKIP_PIP_INSTALL=1
-
-        # Pip freeze, for compatibility.
-        /app/.heroku/python/bin/pip freeze > requirements.txt
-
+        /app/.heroku/python/bin/pipenv lock --requirements --no-hashes > $BUILD_DIR/requirements.txt 2> /dev/null
     fi
 fi

bin/steps/setuptools

@@ -3,9 +3,9 @@
 # Syntax sugar.
 source $BIN_DIR/utils
 
-puts-step "Removing packaging utilities from requirements.txt."
+if (pip-grep -s requirements.txt setuptools distribute &> /dev/null) then
+
+  puts-warn 'The package setuptools/distribute is listed in requirements.txt.'
+  puts-warn 'Please remove to ensure expected behavior. '
 
-if [[ -f requirements.txt ]]; then
-    pip-clean requirements.txt
 fi
-

bin/utils

@@ -3,7 +3,7 @@ shopt -s extglob
 
 # The standard library.
 if [[ ! -f  /tmp/stdlib.sh ]]; then
-  curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
+  curl --retry 3 -s https://raw.githubusercontent.com/heroku/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
 fi
 source /tmp/stdlib.sh
 

builds/README.md

@@ -1,7 +1,5 @@
 # Python Buildpack Binaries
 
-For Cedar-14 stack
-------------------
 
 To get started with it, create an app on Heroku inside a clone of this repository, and set your S3 config vars:
 
@@ -30,12 +28,4 @@ If this works, run `bob deploy` instead of `bob build` to have the result upload
 
 To speed things up drastically, it'll usually be a good idea to `heroku run bash --size PX` instead.
 
-For Heroku-16 stack
--------------------
-
-1. Ensure GNU Make and Docker are installed.
-2. From the root of the buildpack repository, run: `make buildenv-heroku-16`
-3. Follow the instructions displayed!
-
-
-Enjoy :)
+Enjoy :)

builds/runtimes/pypy-5.7.1

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
-
-# NOTICE: This formula only works for the cedar-14 stack, not cedar.
-
-OUT_PREFIX=$1
-
-echo "Building PyPy..."
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy2-v5.7.1-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/pypy3-5.7.1

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
-
-OUT_PREFIX=$1
-
-echo "Building PyPy..."
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy3-v5.7.1-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python

requirements.txt

@@ -1 +1 @@
-bob-builder==0.0.13
+bob-builder==0.0.5

test/utils

@@ -93,14 +93,8 @@ assertNotCaptured()
 
 assertCapturedSuccess()
 {
-  assertEquals "Captured exit code -" "0" "${RETURN}"
-  assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
-
-  if [ $RETURN -ne 0 -a -z "$(cat ${STD_ERR})" ]; then
-    # Failing exit code but stderr was empty. Display stdout to help debugging.
-    cat $STD_OUT
-    echo
-  fi
+  assertEquals "Expected captured exit code to be 0; was <${RETURN}>" "0" "${RETURN}"
+  assertEquals "Expected STD_ERR to be empty; was <$(cat ${STD_ERR})>" "" "$(cat ${STD_ERR})"
 }
 
 # assertCapturedError [[expectedErrorCode] expectedErrorMsg]

vendor/WEB_CONCURRENCY.sh

@@ -1,29 +0,0 @@
-case $(ulimit -u) in
-
-# Automatic configuration for Gunicorn's Workers setting.
-
-# Standard-1X (+Free, +Hobby) Dyno
-256)
-  export DYNO_RAM=512
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-02}
-  ;;
-
-# Standard-2X Dyno
-512)
-  export DYNO_RAM=1024
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-04}
-  ;;
-
-# Performance-M Dyno
-16384)
-  export DYNO_RAM=2560
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-08}
-  ;;
-
-# Performance-L Dyno
-32768)
-  export DYNO_RAM=6656
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-011}
-  ;;
-
-esac

vendor/pip-pop/pip-clean

@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""Usage:
-  pip-clean <req-file>
-
-Options:
-  -h --help     Show this screen.
-"""
-from docopt import docopt
-
-BAD_PACKAGES = ['appdirs', 'packaging', 'pyparsing', 'six', 'setuptools', 'distribute']
-
-
-def good_package(line):
-    package_name = line.split('=')[0].split('<')[0].split('>')[0].split(' ')[0].split('#')[0].split('\n')[0]
-    return package_name not in BAD_PACKAGES
-
-def main():
-    args = docopt(__doc__, version='pip-clean')
-    req_file = args['<req-file>']
-
-    with open(req_file, 'r') as f:
-        # Iterate over every line in the requirements file.
-        lines = [line for line in f if good_package(line)]
-
-    # Write the requirements file to disk.
-    with open(req_file, 'w') as f:
-        f.write(''.join(lines))
-
-
-if __name__ == '__main__':
-    main()

vendor/pip-pop/requirements.txt

@@ -1,17 +0,0 @@
-certifi==2017.4.17
-chardet==3.0.3
-dateparser
-humanize==0.5.1
-idna==2.5
-maya==0.3.2
-pendulum==1.2.1
-python-dateutil==2.6.0
-pytz==2017.2
-pytzdata==2017.2
-regex==2017.5.26
-requests==2.17.3
-ruamel.ordereddict==0.4.9
-ruamel.yaml==0.14.12
-tzlocal==1.4
-urllib3==1.21.1
-

vendor/pip-pop/sp-grep

@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""Usage:
-  sp-grep [-s] <package>...
-
-Options:
-  -h --help     Show this screen.
-"""
-from docopt import docopt
-from pkg_resources import DistributionNotFound, get_distribution
-
-
-def has_any_distribution(names, silent=False):
-    for name in names:
-        try:
-            get_distribution(name)
-        except DistributionNotFound:
-            continue
-
-        if not silent:
-            print('Package {name} found!'.format(name=name))
-
-        exit(0)
-
-    if not silent:
-        print('Not found.')
-
-    exit(1)
-
-
-def main():
-    args = docopt(__doc__, version='sp-grep')
-    has_any_distribution(names=args['<package>'], silent=args['-s'])
-
-
-if __name__ == '__main__':
-    main()

vendor/python.gunicorn.sh

@@ -1,2 +1,38 @@
+if [[ "${WEB_CONCURRENCY:-}" == 0* ]]; then
+  # another buildpack set a default value, with leading zero
+  unset WEB_CONCURRENCY
+fi
+
+case $(ulimit -u) in
+
+# Automatic configuration for Gunicorn's Workers setting.
+# Leading zero padding so a subsequent buildpack can figure out that we set a value, and not the user
+
+# Standard-1X (+Free, +Hobby) Dyno
+256)
+  export DYNO_RAM=512
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-02}
+  ;;
+
+# Standard-2X Dyno
+512)
+  export DYNO_RAM=1024
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-04}
+  ;;
+
+# Performance-M Dyno
+16384)
+  export DYNO_RAM=2560
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-08}
+  ;;
+
+# Performance-L Dyno
+32768)
+  export DYNO_RAM=6656
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-011}
+  ;;
+
+esac
+
 # Automatic configuration for Gunicorn's ForwardedAllowIPS setting.
 export FORWARDED_ALLOW_IPS='*'