Fix pyyaml (#402)

* Revert "use pkg_resources to check for distributions (#395)"

This reverts commit 9b185f99d5.

* use sp-grep for django detection

* sp-grep

* fix setuptools script

CHANGELOG.md

@@ -1,9 +1,22 @@
 # Python Buildpack Changelog
 
+# 106
+
+Don't install packages that could mess up packaging.
+
+- The Python buildpack will automatically remove `six`, `pyparsing`, `appdirs`,
+  `setuptools`, and `distribute` from a `requirements.txt` file now, as these
+  packages are provided by the Python buildpack.
+
+# 105
+
+Improvements to output messaging.
+
 # 104
 
-unreleased
+General improvements.
 
+- Fix for Heroku CI.
 - Use `pkg_resources` to check if a distribution is installed instead of
   parsing `requirements.txt`. ([#395][395])
 

bin/compile

@@ -145,11 +145,6 @@ let start=$(nowms)
 source $BIN_DIR/steps/python
 mtime "python.install.time" "${start}"
 
-# Sanity check for setuptools/distribute.
-let start=$(nowms)
-source $BIN_DIR/steps/setuptools
-mtime "setuptools.install.time" "${start}"
-
 # Pipenv support.
 source $BIN_DIR/steps/pipenv
 
@@ -176,15 +171,19 @@ sub-env $BIN_DIR/steps/geo-libs
 # GDAL support.
 source $BIN_DIR/steps/gdal
 
+# Uninstall removed dependencies with Pip.
+let start=$(nowms)
+source $BIN_DIR/steps/pip-uninstall
+mtime "pip.uninstall.time" "${start}"
+
+# Cleanup requirements.txt
+source $BIN_DIR/steps/setuptools
+
 # Install dependencies with Pip (where the magic happens).
 let start=$(nowms)
 source $BIN_DIR/steps/pip-install
 mtime "pip.install.time" "${start}"
 
-# Uninstall removed dependencies with Pip.
-let start=$(nowms)
-source $BIN_DIR/steps/pip-uninstall
-mtime "pip.uninstall.time" "${start}"
 
 # Support for NLTK corpora.
 let start=$(nowms)

bin/steps/collectstatic

@@ -20,7 +20,7 @@ MANAGE_FILE=${MANAGE_FILE:-fakepath}
 [ -f .heroku/collectstatic_disabled ] && DISABLE_COLLECTSTATIC=1
 
 # Ensure that Django is explicitly specified in requirements.txt
-pip-grep -s Django && DJANGO_INSTALLED=1
+sp-grep django && DJANGO_INSTALLED=1
 
 
 if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then

bin/steps/cryptography

@@ -18,7 +18,7 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 source $BIN_DIR/utils
 
 # If a package using cffi exists within requirements, use vendored libffi.
-if (pip-grep -s argon2-cffi bcrypt cffi cryptography PyNaCl pyOpenSSL PyOpenSSL misaka &> /dev/null) then
+if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[argon2] Django[argon2] django[bcrypt] Django[bcrypt] PyNaCl pyOpenSSL PyOpenSSL requests[security] misaka &> /dev/null) then
 
   if [ ! -d ".heroku/vendor/lib/libffi-3.1" ]; then
     echo "-----> Noticed cffi. Bootstrapping libffi."

bin/steps/gdal

@@ -18,7 +18,7 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 source $BIN_DIR/utils
 
 # If GDAL exists within requirements, use vendored gdal.
-if (pip-grep -s GDAL pygdal &> /dev/null) then
+if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
 
   if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then
     echo "-----> Noticed GDAL. Bootstrapping gdal."

bin/steps/pylibmc

@@ -17,7 +17,7 @@ source $BIN_DIR/utils
 
 
 # If pylibmc exists within requirements, use vendored libmemcached.
-if (pip-grep -s pylibmc &> /dev/null) then
+if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
 
   if [ ! -d ".heroku/vendor/lib/sasl2" ]; then
     echo "-----> Noticed pylibmc. Bootstrapping libmemcached."

bin/steps/setuptools

@@ -3,9 +3,9 @@
 # Syntax sugar.
 source $BIN_DIR/utils
 
-if (pip-grep -s setuptools distribute &> /dev/null) then
-
-  puts-warn 'The package setuptools/distribute is listed in requirements.txt.'
-  puts-warn 'Please remove to ensure expected behavior. '
+puts-step "Removing packaging utilities from requirements.txt."
 
+if [[ -f requirements.txt ]]; then
+    pip-clean requirements.txt
 fi
+

requirements.txt

@@ -1,2 +1 @@
-# TODO: Replace this once the bob-builder changes on master are released:
-https://github.com/kennethreitz/bob-builder/archive/54211376a8fb49c67ecbd6798bd45b55f4d125f4.zip
+bob-builder==0.0.13

vendor/pip-pop/pip-clean

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Usage:
+  pip-clean <req-file>
+
+Options:
+  -h --help     Show this screen.
+"""
+from docopt import docopt
+
+BAD_PACKAGES = ['appdirs', 'packaging', 'pyparsing', 'six', 'setuptools', 'distribute']
+
+
+def good_package(line):
+    package_name = line.split('=')[0].split('<')[0].split('>')[0].split(' ')[0].split('#')[0].split('\n')[0]
+    return package_name not in BAD_PACKAGES
+
+def main():
+    args = docopt(__doc__, version='pip-clean')
+    req_file = args['<req-file>']
+
+    with open(req_file, 'r') as f:
+        # Iterate over every line in the requirements file.
+        lines = [line for line in f if good_package(line)]
+
+    # Write the requirements file to disk.
+    with open(req_file, 'w') as f:
+        f.write(''.join(lines))
+
+
+if __name__ == '__main__':
+    main()

vendor/pip-pop/pip-grep

@@ -2,26 +2,58 @@
 # -*- coding: utf-8 -*-
 
 """Usage:
-  pip-grep [-s] <package>...
+  pip-grep [-s] <reqfile> <package>...
 
 Options:
   -h --help     Show this screen.
 """
+import os
 from docopt import docopt
-from pkg_resources import DistributionNotFound, get_distribution
+from pip.req import parse_requirements
+from pip.index import PackageFinder
+from pip._vendor.requests import session
+
+requests = session()
 
 
-def has_any_distribution(names, silent=False):
-    for name in names:
-        try:
-            get_distribution(name)
-        except DistributionNotFound:
-            continue
+class Requirements(object):
+    def __init__(self, reqfile=None):
+        super(Requirements, self).__init__()
+        self.path = reqfile
+        self.requirements = []
 
+        if reqfile:
+            self.load(reqfile)
+
+    def __repr__(self):
+        return '<Requirements \'{}\'>'.format(self.path)
+
+    def load(self, reqfile):
+        if not os.path.exists(reqfile):
+            raise ValueError('The given requirements file does not exist.')
+
+        finder = PackageFinder([], [], session=requests)
+        for requirement in parse_requirements(reqfile, finder=finder, session=requests):
+            if requirement.req:
+                if not getattr(requirement.req, 'name', None):
+                    # Prior to pip 8.1.2 the attribute `name` did not exist.
+                    requirement.req.name = requirement.req.project_name
+                self.requirements.append(requirement.req)
+
+
+def grep(reqfile, packages, silent=False):
+    try:
+        r = Requirements(reqfile)
+    except ValueError:
         if not silent:
-            print('Package {name} found!'.format(name=name))
+            print('There was a problem loading the given requirement file.')
+        exit(os.EX_NOINPUT)
 
-        exit(0)
+    for req in r.requirements:
+        if req.name in packages:
+            if not silent:
+                print('Package {} found!'.format(req.name))
+            exit(0)
 
     if not silent:
         print('Not found.')
@@ -31,7 +63,10 @@ def has_any_distribution(names, silent=False):
 
 def main():
     args = docopt(__doc__, version='pip-grep')
-    has_any_distribution(names=args['<package>'], silent=args['-s'])
+
+    kwargs = {'reqfile': args['<reqfile>'], 'packages': args['<package>'], 'silent': args['-s']}
+
+    grep(**kwargs)
 
 
 if __name__ == '__main__':

vendor/pip-pop/requirements.txt

@@ -0,0 +1,17 @@
+certifi==2017.4.17
+chardet==3.0.3
+dateparser
+humanize==0.5.1
+idna==2.5
+maya==0.3.2
+pendulum==1.2.1
+python-dateutil==2.6.0
+pytz==2017.2
+pytzdata==2017.2
+regex==2017.5.26
+requests==2.17.3
+ruamel.ordereddict==0.4.9
+ruamel.yaml==0.14.12
+tzlocal==1.4
+urllib3==1.21.1
+

vendor/pip-pop/sp-grep

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Usage:
+  sp-grep [-s] <package>...
+
+Options:
+  -h --help     Show this screen.
+"""
+from docopt import docopt
+from pkg_resources import DistributionNotFound, get_distribution
+
+
+def has_any_distribution(names, silent=False):
+    for name in names:
+        try:
+            get_distribution(name)
+        except DistributionNotFound:
+            continue
+
+        if not silent:
+            print('Package {name} found!'.format(name=name))
+
+        exit(0)
+
+    if not silent:
+        print('Not found.')
+
+    exit(1)
+
+
+def main():
+    args = docopt(__doc__, version='sp-grep')
+    has_any_distribution(names=args['<package>'], silent=args['-s'])
+
+
+if __name__ == '__main__':
+    main()