add retry to curl

.dockerignore

@@ -1 +0,0 @@
-.git/

CHANGELOG.md

@@ -1,25 +1,5 @@
 # Python Buildpack Changelog
 
-# 104
-
-General improvements.
-
-- Fix for Heroku CI.
-- Use `pkg_resources` to check if a distribution is installed instead of
-  parsing `requirements.txt`. ([#395][395])
-
-[395]: https://github.com/heroku/heroku-buildpack-python/pull/395
-
-## 103
-
-Bug fixes and improvements.
-
-- Fix for Pipenv.
-- Fix for Heroku CI.
-- Improve handling of WEB_CONCURRENCY when using multiple buildpacks.
-- Adjust environment variables set during the build to more closely match those in the dyno environment (DYNO is now available, STACK is not).
-- Restore the build cache prior to running bin/pre_compile.
-
 ## 102
 
 Buildpack code cleanup.

Dockerfile

@@ -1,13 +1,11 @@
 FROM heroku/heroku:16-build
 
 WORKDIR /app
-ENV WORKSPACE_DIR="/app/builds" \
+ENV WORKSPACE_DIR=/app/builds
-    S3_BUCKET="lang-python" \
-    S3_PREFIX="heroku-16/"
 
-RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y python-pip
 
-COPY requirements.txt /app/
+# Install bob-builder application
-RUN pip install --disable-pip-version-check --no-cache-dir -r /app/requirements.txt
+RUN pip install bob-builder==0.0.5
 
-COPY . /app
+COPY . /app

Makefile

@@ -13,20 +13,7 @@ test-heroku-16:
 	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
 	@echo ""
 
-buildenv-heroku-16:
-	@echo "Creating build environment (heroku-16)..."
-	@echo
-	@docker build --pull -t python-buildenv-heroku-16 .
-	@echo
-	@echo "Usage..."
-	@echo
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ bob build runtimes/python-2.7.13"
-	@echo "  $$ bob deploy runtimes/python-2.7.13"
-	@echo
-	@docker run -it --rm python-buildenv-heroku-16
-
 tools:
 	git clone https://github.com/kennethreitz/pip-pop.git
 	mv pip-pop/bin/* vendor/pip-pop/
-	rm -fr pip-pop
+	rm -fr pip-pop

README.md

@@ -52,5 +52,5 @@ Runtime options include:
 
 - `python-2.7.13`
 - `python-3.6.1`
-- `pypy-5.7.1` (unsupported, experimental)
+- `pypy-5.7.0` (unsupported, experimental)
-- `pypy3-5.5.1` (unsupported, experimental)
+- `pypy3-5.5.0` (unsupported, experimental)

bin/compile

@@ -68,7 +68,6 @@ mkdir -p /app/.heroku
 PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
 EXPORT_PATH="$BIN_DIR/../export"
 GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh"
-WEB_CONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/WEB_CONCURRENCY.sh"
 
 # We'll need to send these statics to other scripts we `source`.
 export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH
@@ -92,20 +91,6 @@ if [[ ! -f Procfile ]]; then
   puts-warn 'Learn more: https://devcenter.heroku.com/articles/procfile'
 fi
 
-# Prepare the cache.
-mkdir -p $CACHE_DIR
-
-# Restore old artifacts from the cache.
-mkdir -p .heroku
-
-cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
-if [[ -d $CACHE_DIR/.heroku/src ]]; then
-  cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
-fi
-
 # Experimental pre_compile hook.
 source $BIN_DIR/steps/hooks/pre_compile
 
@@ -129,6 +114,21 @@ if [ ! -f runtime.txt ]; then
   echo $DEFAULT_PYTHON_VERSION > runtime.txt
 fi
 
+# Prepare the cache.
+mkdir -p $CACHE_DIR
+
+# Restore old artifacts from the cache.
+mkdir -p .heroku
+
+cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
+cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
+if [[ -d $CACHE_DIR/.heroku/src ]]; then
+  cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
+fi
+
+
 mkdir -p $(dirname $PROFILE_PATH)
 mkdir -p /app/.heroku/src
 
@@ -192,12 +192,8 @@ sub-env $BIN_DIR/steps/nltk
 mtime "nltk.download.time" "${start}"
 
 # Support for pip install -e.
-# In CI, $BUILD_DIR is /app.
+rm -fr $BUILD_DIR/.heroku/src
-if [[ ! "$BUILD_DIR" == "/app" ]]; then
+deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
-  rm -fr $BUILD_DIR/.heroku/src
-  deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src
-fi
-
 
 # Django collectstatic support.
 let start=$(nowms)
@@ -215,10 +211,8 @@ set-default-env PYTHONHASHSEED random
 set-default-env PYTHONPATH /app/
 
 # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS.
-cp $ROOT_DIR/vendor/WEB_CONCURRENCY.sh $WEB_CONCURRENCY_PROFILE_PATH
 cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH
 
-
 # Experimental post_compile hook.
 source $BIN_DIR/steps/hooks/post_compile
 

bin/steps/collectstatic

@@ -20,7 +20,7 @@ MANAGE_FILE=${MANAGE_FILE:-fakepath}
 [ -f .heroku/collectstatic_disabled ] && DISABLE_COLLECTSTATIC=1
 
 # Ensure that Django is explicitly specified in requirements.txt
-pip-grep -s Django && DJANGO_INSTALLED=1
+pip-grep -s requirements.txt django Django && DJANGO_INSTALLED=1
 
 
 if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then

bin/steps/cryptography

@@ -18,7 +18,7 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 source $BIN_DIR/utils
 
 # If a package using cffi exists within requirements, use vendored libffi.
-if (pip-grep -s argon2-cffi bcrypt cffi cryptography PyNaCl pyOpenSSL PyOpenSSL misaka &> /dev/null) then
+if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[argon2] Django[argon2] django[bcrypt] Django[bcrypt] PyNaCl pyOpenSSL PyOpenSSL requests[security] misaka &> /dev/null) then
 
   if [ ! -d ".heroku/vendor/lib/libffi-3.1" ]; then
     echo "-----> Noticed cffi. Bootstrapping libffi."

bin/steps/gdal

@@ -18,7 +18,7 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 source $BIN_DIR/utils
 
 # If GDAL exists within requirements, use vendored gdal.
-if (pip-grep -s GDAL pygdal &> /dev/null) then
+if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
 
   if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then
     echo "-----> Noticed GDAL. Bootstrapping gdal."

bin/steps/pip-install

@@ -1,23 +1,20 @@
-if [ ! "$SKIP_PIP_INSTALL" ]; then
+# Install dependencies with Pip.
+puts-step "Installing requirements with pip"
 
-    # Install dependencies with Pip.
+set +e
-    puts-step "Installing requirements with pip"
+/app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
+PIP_STATUS="${PIPESTATUS[0]}"
+set -e
 
-    set +e
+show-warnings
-    /app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
-    PIP_STATUS="${PIPESTATUS[0]}"
-    set -e
 
-    show-warnings
+if [[ ! $PIP_STATUS -eq 0 ]]; then
-
+    exit 1
-    if [[ ! $PIP_STATUS -eq 0 ]]; then
+fi
-        exit 1
-    fi
 
 
-    # Smart Requirements handling
+# Smart Requirements handling
-    cp requirements.txt .heroku/python/requirements-declared.txt
+cp requirements.txt .heroku/python/requirements-declared.txt
-    /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
+/app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
 
-    echo
+echo
-fi

bin/steps/pipenv

@@ -1,19 +1,10 @@
 # Pipenv support (Generate requriements.txt with pipenv).
 if [[ -f Pipfile ]]; then
     if [[ ! -f requirements.txt ]]; then
-        puts-step "Installing requirements with latest pipenv..."
+        puts-step "Generating 'requirements.txt' with pipenv"
 
-        # Install pipenv.
         /app/.heroku/python/bin/pip install pipenv --upgrade &> /dev/null
 
-        # Install the dependencies.
+        /app/.heroku/python/bin/pipenv lock --requirements --no-hashes > $BUILD_DIR/requirements.txt 2> /dev/null
-        /app/.heroku/python/bin/pipenv install --system 2>&1 | indent
-
-        # Skip pip install, later.
-        export SKIP_PIP_INSTALL=1
-
-        # Pip freeze, for compatibility.
-        /app/.heroku/python/bin/pip freeze > requirements.txt
-
     fi
 fi

bin/steps/pylibmc

@@ -17,7 +17,7 @@ source $BIN_DIR/utils
 
 
 # If pylibmc exists within requirements, use vendored libmemcached.
-if (pip-grep -s pylibmc &> /dev/null) then
+if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
 
   if [ ! -d ".heroku/vendor/lib/sasl2" ]; then
     echo "-----> Noticed pylibmc. Bootstrapping libmemcached."

bin/steps/setuptools

@@ -3,7 +3,7 @@
 # Syntax sugar.
 source $BIN_DIR/utils
 
-if (pip-grep -s setuptools distribute &> /dev/null) then
+if (pip-grep -s requirements.txt setuptools distribute &> /dev/null) then
 
   puts-warn 'The package setuptools/distribute is listed in requirements.txt.'
   puts-warn 'Please remove to ensure expected behavior. '

bin/utils

@@ -3,7 +3,7 @@ shopt -s extglob
 
 # The standard library.
 if [[ ! -f  /tmp/stdlib.sh ]]; then
-  curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
+  curl --retry 3 -s https://raw.githubusercontent.com/heroku/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh
 fi
 source /tmp/stdlib.sh
 

builds/README.md

@@ -1,7 +1,5 @@
 # Python Buildpack Binaries
 
-For Cedar-14 stack
-------------------
 
 To get started with it, create an app on Heroku inside a clone of this repository, and set your S3 config vars:
 
@@ -30,12 +28,4 @@ If this works, run `bob deploy` instead of `bob build` to have the result upload
 
 To speed things up drastically, it'll usually be a good idea to `heroku run bash --size PX` instead.
 
-For Heroku-16 stack
+Enjoy :)
--------------------
-
-1. Ensure GNU Make and Docker are installed.
-2. From the root of the buildpack repository, run: `make buildenv-heroku-16`
-3. Follow the instructions displayed!
-
-
-Enjoy :)

builds/runtimes/pypy-5.7.1

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
-
-# NOTICE: This formula only works for the cedar-14 stack, not cedar.
-
-OUT_PREFIX=$1
-
-echo "Building PyPy..."
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy2-v5.7.1-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/pypy3-5.7.1

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-# Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
-
-OUT_PREFIX=$1
-
-echo "Building PyPy..."
-SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-linux64.tar.bz2'
-curl -L $SOURCE_TARBALL | tar jx
-cp -R pypy3-v5.7.1-linux64/* $OUT_PREFIX
-
-ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python

requirements.txt

@@ -1,2 +1 @@
-# TODO: Replace this once the bob-builder changes on master are released:
+bob-builder==0.0.5
-https://github.com/kennethreitz/bob-builder/archive/54211376a8fb49c67ecbd6798bd45b55f4d125f4.zip

test/utils

@@ -93,14 +93,8 @@ assertNotCaptured()
 
 assertCapturedSuccess()
 {
-  assertEquals "Captured exit code -" "0" "${RETURN}"
+  assertEquals "Expected captured exit code to be 0; was <${RETURN}>" "0" "${RETURN}"
-  assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})"
+  assertEquals "Expected STD_ERR to be empty; was <$(cat ${STD_ERR})>" "" "$(cat ${STD_ERR})"
-
-  if [ $RETURN -ne 0 -a -z "$(cat ${STD_ERR})" ]; then
-    # Failing exit code but stderr was empty. Display stdout to help debugging.
-    cat $STD_OUT
-    echo
-  fi
 }
 
 # assertCapturedError [[expectedErrorCode] expectedErrorMsg]

vendor/WEB_CONCURRENCY.sh

@@ -1,29 +0,0 @@
-case $(ulimit -u) in
-
-# Automatic configuration for Gunicorn's Workers setting.
-
-# Standard-1X (+Free, +Hobby) Dyno
-256)
-  export DYNO_RAM=512
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-02}
-  ;;
-
-# Standard-2X Dyno
-512)
-  export DYNO_RAM=1024
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-04}
-  ;;
-
-# Performance-M Dyno
-16384)
-  export DYNO_RAM=2560
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-08}
-  ;;
-
-# Performance-L Dyno
-32768)
-  export DYNO_RAM=6656
-  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-011}
-  ;;
-
-esac

vendor/pip-pop/pip-grep

@@ -2,26 +2,58 @@
 # -*- coding: utf-8 -*-
 
 """Usage:
-  pip-grep [-s] <package>...
+  pip-grep [-s] <reqfile> <package>...
 
 Options:
   -h --help     Show this screen.
 """
+import os
 from docopt import docopt
-from pkg_resources import DistributionNotFound, get_distribution
+from pip.req import parse_requirements
+from pip.index import PackageFinder
+from pip._vendor.requests import session
+
+requests = session()
 
 
-def has_any_distribution(names, silent=False):
+class Requirements(object):
-    for name in names:
+    def __init__(self, reqfile=None):
-        try:
+        super(Requirements, self).__init__()
-            get_distribution(name)
+        self.path = reqfile
-        except DistributionNotFound:
+        self.requirements = []
-            continue
 
+        if reqfile:
+            self.load(reqfile)
+
+    def __repr__(self):
+        return '<Requirements \'{}\'>'.format(self.path)
+
+    def load(self, reqfile):
+        if not os.path.exists(reqfile):
+            raise ValueError('The given requirements file does not exist.')
+
+        finder = PackageFinder([], [], session=requests)
+        for requirement in parse_requirements(reqfile, finder=finder, session=requests):
+            if requirement.req:
+                if not getattr(requirement.req, 'name', None):
+                    # Prior to pip 8.1.2 the attribute `name` did not exist.
+                    requirement.req.name = requirement.req.project_name
+                self.requirements.append(requirement.req)
+
+
+def grep(reqfile, packages, silent=False):
+    try:
+        r = Requirements(reqfile)
+    except ValueError:
         if not silent:
-            print('Package {name} found!'.format(name=name))
+            print('There was a problem loading the given requirement file.')
+        exit(os.EX_NOINPUT)
 
-        exit(0)
+    for req in r.requirements:
+        if req.name in packages:
+            if not silent:
+                print('Package {} found!'.format(req.name))
+            exit(0)
 
     if not silent:
         print('Not found.')
@@ -31,7 +63,10 @@ def has_any_distribution(names, silent=False):
 
 def main():
     args = docopt(__doc__, version='pip-grep')
-    has_any_distribution(names=args['<package>'], silent=args['-s'])
+
+    kwargs = {'reqfile': args['<reqfile>'], 'packages': args['<package>'], 'silent': args['-s']}
+
+    grep(**kwargs)
 
 
 if __name__ == '__main__':

vendor/python.gunicorn.sh

@@ -1,2 +1,38 @@
+if [[ "${WEB_CONCURRENCY:-}" == 0* ]]; then
+  # another buildpack set a default value, with leading zero
+  unset WEB_CONCURRENCY
+fi
+
+case $(ulimit -u) in
+
+# Automatic configuration for Gunicorn's Workers setting.
+# Leading zero padding so a subsequent buildpack can figure out that we set a value, and not the user
+
+# Standard-1X (+Free, +Hobby) Dyno
+256)
+  export DYNO_RAM=512
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-02}
+  ;;
+
+# Standard-2X Dyno
+512)
+  export DYNO_RAM=1024
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-04}
+  ;;
+
+# Performance-M Dyno
+16384)
+  export DYNO_RAM=2560
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-08}
+  ;;
+
+# Performance-L Dyno
+32768)
+  export DYNO_RAM=6656
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-011}
+  ;;
+
+esac
+
 # Automatic configuration for Gunicorn's ForwardedAllowIPS setting.
 export FORWARDED_ALLOW_IPS='*'