WEBCONCURRENCY_PROFILE_PATH

Update Readme.md to pypy3-2.4.0

Readme.md

@@ -40,12 +40,13 @@ Specify a Runtime
 You can also provide arbitrary releases Python with a `runtime.txt` file.
 
     $ cat runtime.txt
-    python-3.4.1
+    python-3.4.2
 
 Runtime options include:
 
 - python-2.7.8
-- python-3.4.1
+- python-3.4.2
-- pypy-1.9 (experimental)
+- pypy-2.4.0 (unsupported, experimental)
+- pypy3-2.4.0 (unsupported, experimental)
 
 Other [unsupported runtimes](https://github.com/heroku/heroku-buildpack-python/tree/master/builds/runtimes) are available as well.

bin/compile

@@ -24,12 +24,13 @@ CACHED_DIRS=".heroku"
 VIRTUALENV_LOC=".heroku/venv"
 LEGACY_TRIGGER="lib/python2.7"
 PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh"
+WEBCONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/python.webconcurrency.sh"
 
-DEFAULT_PYTHON_VERSION="python-2.7.8"
+DEFAULT_PYTHON_VERSION="python-2.7.9"
 DEFAULT_PYTHON_STACK="cedar"
 PYTHON_EXE="/app/.heroku/python/bin/python"
-PIP_VERSION="1.5.6"
+PIP_VERSION="6.0.6"
-SETUPTOOLS_VERSION="5.4.1"
+SETUPTOOLS_VERSION="11.3.1"
 
 # Setup bpwatch
 export PATH=$PATH:$ROOT_DIR/vendor/bpwatch
@@ -150,6 +151,9 @@ mkdir -p $(dirname $PROFILE_PATH)
 # Install Python.
 source $BIN_DIR/steps/python
 
+# Sanity check for setuptools/distribute.
+source $BIN_DIR/steps/setuptools
+
 # Uninstall removed dependencies with Pip.
 source $BIN_DIR/steps/pip-uninstall
 
@@ -163,7 +167,7 @@ source $BIN_DIR/steps/pylibmc
 source $BIN_DIR/steps/cryptography
 
 # Install dependencies with Pip.
-source $BIN_DIR/steps/pip-install
+sub-env $BIN_DIR/steps/pip-install
 
 # Django collectstatic support.
 sub-env $BIN_DIR/steps/collectstatic
@@ -182,6 +186,9 @@ set-default-env LANG en_US.UTF-8
 set-default-env PYTHONHASHSEED random
 set-default-env PYTHONPATH /app/
 
+# Install sane-default script for WEB_CONCURRENCY environment variable.
+cp $ROOT_DIR/vendor/python.webconcurrency.sh $WEBCONCURRENCY_PROFILE_PATH
+
 
 # Experimental post_compile hook.
 bpwatch start post_compile

bin/steps/cryptography

@@ -19,10 +19,8 @@ source $BIN_DIR/utils
 
 bpwatch start libffi_install
 
-whereis pkg-config
-
 # If pylibmc exists within requirements, use vendored cryptography.
-if (pip-grep -s requirements.txt cffi pylibmc) then
+if (pip-grep -s requirements.txt bcrypt cffi cryptography &> /dev/null) then
 
   if [ -d ".heroku/vendor/lib/libffi-3.1.1" ]; then
     export LIBFFI=$(pwd)/vendor

bin/steps/pip-install

@@ -1,3 +1,7 @@
+#!/usr/bin/env bash
+
+source $BIN_DIR/utils
+
 # Install dependencies with Pip.
 puts-step "Installing dependencies with pip"
 

bin/steps/pylibmc

@@ -19,7 +19,7 @@ source $BIN_DIR/utils
 bpwatch start pylibmc_install
 
 # If pylibmc exists within requirements, use vendored libmemcached.
-if (pip-grep -s requirements.txt pylibmc) then
+if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
 
   if [ -d ".heroku/vendor/lib/sasl2" ]; then
     export LIBMEMCACHED=$(pwd)/vendor

bin/steps/python

@@ -16,7 +16,7 @@ fi
 if [ ! $STACK = $CACHED_PYTHON_STACK ]; then
   bpwatch start uninstall_python
     puts-step "Stack changed, re-installing runtime"
-    rm -fr .heroku/python
+    rm -fr .heroku/python .heroku/python-stack .heroku/vendor
     unset SKIP_INSTALL
   bpwatch stop uninstall_python
 fi
@@ -52,22 +52,22 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_VERSION* ]]; then
 
   bpwatch start prepare_environment
 
+  TMPTARDIR=$(mktemp -d)
+  trap "rm -rf $TMPTARDIR" RETURN
+
   bpwatch start install_setuptools
     # Prepare it for the real world
     # puts-step "Installing Setuptools ($SETUPTOOLS_VERSION)"
-    cd $ROOT_DIR/vendor/
+    tar zxf $ROOT_DIR/vendor/setuptools-$SETUPTOOLS_VERSION.tar.gz -C $TMPTARDIR
-    tar zxf setuptools-$SETUPTOOLS_VERSION.tar.gz
+    cd $TMPTARDIR/setuptools-$SETUPTOOLS_VERSION/
-    cd $ROOT_DIR/vendor/setuptools-$SETUPTOOLS_VERSION/
     python setup.py install &> /dev/null
     cd $WORKING_DIR
   bpwatch stop install_setuptoools
 
   bpwatch start install_pip
     # puts-step "Installing Pip ($PIP_VERSION)"
-
+    tar zxf $ROOT_DIR/vendor/pip-$PIP_VERSION.tar.gz -C $TMPTARDIR
-    cd $ROOT_DIR/vendor/
+    cd $TMPTARDIR/pip-$PIP_VERSION/
-    tar zxf pip-$PIP_VERSION.tar.gz
-    cd $ROOT_DIR/vendor/pip-$PIP_VERSION/
     python setup.py install &> /dev/null
     cd $WORKING_DIR
 

bin/steps/setuptools

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# Syntax sugar.
+source $BIN_DIR/utils
+
+if (pip-grep -s requirements.txt setuptools distribute &> /dev/null) then
+
+  puts-warn 'The package setuptools/distribute is listed in requirements.txt.'
+  puts-warn 'Please remove to ensure expected behavior. '
+
+fi

builds/libraries/libffi

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+
+# Use new path, containing autoconf.
+export PATH="/app/.heroku/python/bin/:$PATH"
+hash -r
+
+
+echo "Building libffi..."
+
+SOURCE_TARBALL='http://cl.ly/2s1t1u3v0N0I/download/libffi-3.1.tar'
+
+curl -L $SOURCE_TARBALL | tar x
+
+cd libffi-3.1
+./configure --prefix=$OUT_PREFIX --disable-static &&
+make
+make install
+
+# Cleanup
+cd ..

builds/libraries/vendor/libffi

@@ -10,7 +10,7 @@ hash -r
 
 echo "Building libffi..."
 
-SOURCE_TARBALL='http://cl.ly/2s1t1u3v0N0I/download/libffi-3.1.tar'
+SOURCE_TARBALL='ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz'
 
 curl -L $SOURCE_TARBALL | tar x
 

builds/libraries/vendor/libmemcache

@@ -22,7 +22,8 @@ curl -LO ftp://ftp.cyrusimap.org/cyrus-sasl/cyrus-sasl-2.1.26.tar.gz
 tar xzf cyrus-sasl-2.1.26.tar.gz
 
 pushd cyrus-sasl-2.1.26
-./configure --prefix=${OUT_PREFIX}
+./configure --prefix=${OUT_PREFIX} --with-plugindir=${OUT_PREFIX}lib/sasl2 --with-configdir=${OUT_PREFIX}lib/sasl2
+
 make -s -j 9
 make install -s
 popd

builds/runtimes/pypy-2.3

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
+# Build Deps: libraries/sqlite, libraries/libffi
 
 # NOTICE: This formula only works for the cedar-14 stack, not cedar.
 

builds/runtimes/pypy-2.4.0

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+
+OUT_PREFIX=$1
+
+echo "Building PyPy..."
+SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-linux64.tar.bz2'
+curl -L $SOURCE_TARBALL | tar jx
+cp -R pypy-2.4.0-linux64/* $OUT_PREFIX
+
+ln $OUT_PREFIX/bin/pypy $OUT_PREFIX/bin/python

builds/runtimes/pypy3-2.4.0

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+# NOTICE: This formula only works for the cedar-14 stack, not cedar.
+
+OUT_PREFIX=$1
+
+echo "Building PyPy..."
+SOURCE_TARBALL='https://bitbucket.org/pypy/pypy/downloads/pypy3-2.4.0-linux64.tar.bz2'
+curl -L $SOURCE_TARBALL | tar jx
+cp -R pypy3-2.4.0-linux64/* $OUT_PREFIX
+
+ln $OUT_PREFIX/bin/pypy3 $OUT_PREFIX/bin/python

builds/runtimes/python-2.7.8-shared

@@ -10,6 +10,6 @@ curl -L $SOURCE_TARBALL | tar xz
 mv Python-2.7.8 src
 cd src
 
-./configure --prefix=$OUT_PREFIX
+./configure --prefix=$OUT_PREFIX --enable-shared
 make
 make install

builds/runtimes/python-2.7.9

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python..."
+SOURCE_TARBALL='http://python.org/ftp/python/2.7.9/Python-2.7.9.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-2.7.9 src
+cd src
+
+./configure --prefix=$OUT_PREFIX  --with-ensurepip=no
+make
+make install

builds/runtimes/python-3.3.6

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python..."
+SOURCE_TARBALL='http://python.org/ftp/python/3.3.6/Python-3.3.6.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.3.6 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --enable-shared
+make
+make install
+
+ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python

builds/runtimes/python-3.4.2

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+# Build Deps: libraries/sqlite
+
+OUT_PREFIX=$1
+
+echo "Building Python..."
+SOURCE_TARBALL='http://python.org/ftp/python/3.4.2/Python-3.4.2.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.4.2 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no --enable-shared
+make
+make install
+
+ln $OUT_PREFIX/bin/python3 $OUT_PREFIX/bin/python
+

vendor/pip-pop/pip-diff

@@ -13,6 +13,10 @@ Options:
 import os
 from docopt import docopt
 from pip.req import parse_requirements
+from pip.index import PackageFinder
+from pip._vendor.requests import session
+
+requests = session()
 
 class Requirements(object):
     def __init__(self, reqfile=None):
@@ -31,7 +35,8 @@ class Requirements(object):
         if not os.path.exists(reqfile):
             raise ValueError('The given requirements file does not exist.')
 
-        for requirement in parse_requirements(reqfile):
+        finder = PackageFinder([], [], session=requests)
+        for requirement in parse_requirements(reqfile, finder=finder, session=requests):
             if requirement.req:
                 self.requirements.append(requirement.req)
 
@@ -79,18 +84,18 @@ def diff(r1, r2, include_fresh=False, include_stale=False):
         r1 = Requirements(r1)
         r2 = Requirements(r2)
     except ValueError:
-        print 'There was a problem loading the given requirements files.'
+        print('There was a problem loading the given requirements files.')
         exit(os.EX_NOINPUT)
 
     results = r1.diff(r2, ignore_versions=True)
 
     if include_fresh:
         for line in results['fresh']:
-            print line.project_name if include_versions else line
+            print(line.project_name if include_versions else line)
 
     if include_stale:
         for line in results['stale']:
-            print line.project_name if include_versions else line
+            print(line.project_name if include_versions else line)
 
 
 

vendor/pip-pop/pip-grep

@@ -10,6 +10,10 @@ Options:
 import os
 from docopt import docopt
 from pip.req import parse_requirements
+from pip.index import PackageFinder
+from pip._vendor.requests import session
+
+requests = session()
 
 class Requirements(object):
     def __init__(self, reqfile=None):
@@ -28,7 +32,8 @@ class Requirements(object):
         if not os.path.exists(reqfile):
             raise ValueError('The given requirements file does not exist.')
 
-        for requirement in parse_requirements(reqfile):
+        finder = PackageFinder([], [], session=requests)
+        for requirement in parse_requirements(reqfile, finder=finder, session=requests):
             self.requirements.append(requirement)
 
 
@@ -41,7 +46,7 @@ def grep(reqfile, packages, silent=False):
     except ValueError:
 
         if not silent:
-            print 'There was a problem loading the given requirement file.'
+            print('There was a problem loading the given requirement file.')
 
         exit(os.EX_NOINPUT)
 
@@ -52,12 +57,12 @@ def grep(reqfile, packages, silent=False):
             if requirement.req.project_name in packages:
 
                 if not silent:
-                    print 'Package {} found!'.format(requirement.req.project_name)
+                    print('Package {} found!'.format(requirement.req.project_name))
 
                 exit(0)
 
     if not silent:
-        print 'Not found.'.format(requirement.req.project_name)
+        print('Not found.'.format(requirement.req.project_name))
 
     exit(1)
 

vendor/python.webconcurrency.sh

@@ -0,0 +1,21 @@
+case $(ulimit -u) in
+
+# 1X DYNO
+256)
+  export DYNO_RAM=512
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-2}
+  ;;
+
+# 2X DYNO
+512)
+  export DYNO_RAM=1024
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-4}
+  ;;
+
+# PX DYNO
+32768)
+  export DYNO_RAM=8192
+  export WEB_CONCURRENCY=${WEB_CONCURRENCY:-9}
+  ;;
+
+esac