Merge pull request #769 from heroku/pip-pipenv-pin

Pin pip version used by pipenv

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,8 @@
+<!-- Hi and welcome to the Heroku Python buildpack repository!
+
+If you meant to open a PR against a fork instead of upstream, please adjust the base branch:
+https://help.github.com/articles/changing-the-base-branch-of-a-pull-request/
+
+Otherwise thank you in advance for your Pull Request - just remember to
+include as much information as possible to help the reviewers :-)
+-->

.gitignore

@@ -3,3 +3,7 @@ site
 .DS_Store
 
 /.envrc
+repos/*
+
+#Venv
+buildpack/*

.travis.yml

@@ -1,25 +1,39 @@
-language: bash
+language: ruby
 dist: trusty
+sudo: required
+rvm:
+- 2.4.4
+before_script:
+  - gem install bundler -v 1.16.2
+  - bundle exec rake hatchet:setup_travis
+before_install:
+  - sudo bash etc/ci-setup.sh
 jobs:
   include:
-    - stage: "Bash linting (shellcheck)"
-      sudo: false
-      before_install:
-        - wget -c https://goo.gl/ZzKHFv -O - | tar -xvJ -C /tmp/
-        - PATH="/tmp/shellcheck-latest:$PATH"
-      script: make check
-
-    - stage: "Stack Tests"
-      services: docker
-      env: STACK=heroku-18
-      script: ./tests.sh
-
-    - stage: "Stack Tests"
-      services: docker
-      env: STACK=heroku-16
-      script: ./tests.sh
-
-    - stage: "Stack Tests"
-      services: docker
-      env: STACK=cedar-14
-      script: ./tests.sh
+  - stage: Bash linting (shellcheck)
+    sudo: false
+    before_install:
+    - wget -c https://goo.gl/ZzKHFv -O - | tar -xvJ -C /tmp/
+    - PATH="/tmp/shellcheck-latest:$PATH"
+    script: make check
+  - stage: Stack Unit Tests
+    services: docker
+    env: STACK=heroku-18
+    script: "./tests.sh"
+  - stage: Stack Unit Tests
+    services: docker
+    env: STACK=heroku-16
+    script: "./tests.sh"
+  - stage: Stack Unit Tests
+    services: docker
+    env: STACK=cedar-14
+    script: "./tests.sh"
+  - stage: Hatchet Integration
+    script: "bundle exec rspec"
+env:
+  global:
+  - HATCHET_RETRIES=3
+  - IS_RUNNING_ON_CI=true
+  - HATCHET_APP_LIMIT=5
+  - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
+  - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=

CHANGELOG.md

@@ -1,22 +1,36 @@
 # Python Buildpack Changelog
 
-# 138
+# 141 (2018-10-10)
+
+Switch to cautious upgrade for Pipenv install to ensure the pinned pip version
+is used with Pipenv
+
+# 140 (2018-10-09)
+
+Add support for detecting SLUGIFY_USES_TEXT_UNIDECODE, which is required to
+install Apache Airflow version 1.10 or higher.
+
+# 139 (2018-10-08)
+
+Improvements to Python install messaging
+
+# 138 (2018-08-01)
 
 Use stack image SQLite3 instead of vendoring
 
-# 137
+# 137 (2018-07-17)
 
 Prevent 3.7.0 from appearing as unsupported in buildpack messaging.
 
-# 136
+# 136 (2018-06-28)
 
 Upgrade to 3.6.6 and support 3.7.0 on all runtimes.
 
-# 135
+# 135 (2018-05-29)
 
 Upgrade Pipenv to v2018.5.18.
 
-# 134
+# 134 (2018-05-02)
 
 Default to 3.6.5, bugfixes.
 

Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+gem "rspec"
+gem "heroku_hatchet"
+gem "rspec-retry"
+gem "rake"

Gemfile.lock

@@ -0,0 +1,71 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    concurrent-ruby (1.0.5)
+    diff-lcs (1.3)
+    erubis (2.7.0)
+    excon (0.62.0)
+    heroics (0.0.24)
+      erubis (~> 2.0)
+      excon
+      moneta
+      multi_json (>= 1.9.2)
+    heroku_hatchet (4.0.2)
+      excon (~> 0)
+      minitest-retry (~> 0.1.9)
+      platform-api (~> 2)
+      repl_runner (~> 0.0.3)
+      rrrretry (~> 1)
+      thor (~> 0)
+      threaded (~> 0)
+    i18n (1.1.0)
+      concurrent-ruby (~> 1.0)
+    minitest (5.11.3)
+    minitest-retry (0.1.9)
+      minitest (>= 5.0)
+    moneta (0.8.1)
+    multi_json (1.13.1)
+    platform-api (2.1.0)
+      heroics (~> 0.0.23)
+      moneta (~> 0.8.1)
+    rake (12.3.1)
+    repl_runner (0.0.3)
+      activesupport
+    rrrretry (1.0.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-retry (0.6.1)
+      rspec-core (> 3.3)
+    rspec-support (3.8.0)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    threaded (0.0.4)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  heroku_hatchet
+  rake
+  rspec
+  rspec-retry
+
+BUNDLED WITH
+   1.16.3

Rakefile

@@ -0,0 +1 @@
+require 'hatchet/tasks'

bin/steps/pip-install

@@ -8,10 +8,24 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     # Install dependencies with Pip.
     puts-step "Installing requirements with pip"
 
-    # Set PIP_EXTRA_INDEX_URL
-    if [[ -r $ENV_DIR/PIP_EXTRA_INDEX_URL ]]; then
+    # Set Pip env vars
+    # This reads certain environment variables set on the Heroku app config
+    # and makes them accessible to the pip install process.
+    #
+    # PIP_EXTRA_INDEX_URL allows for an alternate pypi URL to be used.
+    if [[ -r "$ENV_DIR/PIP_EXTRA_INDEX_URL" ]]; then
         PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")"
         export PIP_EXTRA_INDEX_URL
+        mcount "buildvar.PIP_EXTRA_INDEX_URL"
+    fi
+
+    set +e
+
+    # Set SLUGIFY_USES_TEXT_UNIDECODE, required for Airflow versions >=1.10
+    if [[ -r "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE" ]]; then
+        SLUGIFY_USES_TEXT_UNIDECODE="$(cat "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE")"
+        export SLUGIFY_USES_TEXT_UNIDECODE
+        mcount "buildvar.SLUGIFY_USES_TEXT_UNIDECODE"
     fi
 
     set +e

bin/steps/pipenv

@@ -20,7 +20,6 @@ if [[ -f Pipfile.lock ]]; then
                 export SKIP_PIPENV_INSTALL=1
                 export SKIP_PIP_INSTALL=1
             fi
-
         fi
     fi
 fi
@@ -35,16 +34,31 @@ if [ ! "$SKIP_PIPENV_INSTALL" ]; then
         # Skip pip install, later.
         export SKIP_PIP_INSTALL=1
 
-        # Set PIP_EXTRA_INDEX_URL
-        if [[ -r $ENV_DIR/PIP_EXTRA_INDEX_URL ]]; then
+        # Set Pip env vars
+        # This reads certain environment variables set on the Heroku app config
+        # and makes them accessible to the pip install process.
+        #
+        # PIP_EXTRA_INDEX_URL allows for an alternate pypi URL to be used.
+        if [[ -r "$ENV_DIR/PIP_EXTRA_INDEX_URL" ]]; then
             PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")"
             export PIP_EXTRA_INDEX_URL
+            mcount "buildvar.PIP_EXTRA_INDEX_URL"
+        fi
+
+        # Set SLUGIFY_USES_TEXT_UNIDECODE, required for Airflow versions >=1.10
+        if [[ -r "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE" ]]; then
+            SLUGIFY_USES_TEXT_UNIDECODE="$(cat "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE")"
+            export SLUGIFY_USES_TEXT_UNIDECODE
+            mcount "buildvar.SLUGIFY_USES_TEXT_UNIDECODE"
         fi
 
         export PIPENV_VERSION="2018.5.18"
 
         # Install pipenv.
-        /app/.heroku/python/bin/pip install pipenv==$PIPENV_VERSION --upgrade &> /dev/null
+        # Due to weird old pip behavior and pipenv behavior, pipenv upgrades pip
+        # to latest if only --upgrade is specified. Specify upgrade strategy to
+        # avoid this eager behavior.
+        /app/.heroku/python/bin/pip install pipenv==$PIPENV_VERSION --upgrade --upgrade-strategy only-if-needed &> /dev/null
 
         # Install the dependencies.
         if [[ ! -f Pipfile.lock ]]; then

bin/steps/python

@@ -12,17 +12,34 @@ if [[ $PYTHON_VERSION =~ ^python-2 ]]; then
     puts-warn "The latest version of Python 2 is $LATEST_2 (you are using $PYTHON_VERSION, which is unsupported)."
     puts-warn "We recommend upgrading by specifying the latest version ($LATEST_2)."
     echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+  else
+    echo "       Using supported version of Python 2 ($PYTHON_VERSION)"
   fi
 else
-  if [[ $PYTHON_VERSION =~ ^python-3.7 ]] && [[ "$PYTHON_VERSION" != "$LATEST_37" ]]; then
-    puts-warn "The latest version of Python 3.7 is $LATEST_37 (you are using $PYTHON_VERSION, which is unsupported)."
-    puts-warn "We recommend upgrading by specifying the latest version ($LATEST_37)."
-    echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
-  else
-    if [[ "$PYTHON_VERSION" != "$LATEST_36" ]]; then
-      puts-warn "The latest version of Python 3.6 is $LATEST_36 (you are using $PYTHON_VERSION, which is unsupported)."
-      puts-warn "We recommend upgrading by specifying the latest version ($LATEST_36)."
+  if [[ $PYTHON_VERSION =~ ^python-3 ]]; then
+    if [[ $PYTHON_VERSION =~ ^python-3.7 ]]; then
+      if [[ "$PYTHON_VERSION" != "$LATEST_37" ]]; then
+      puts-warn "The latest version of Python 3.7 is $LATEST_37 (you are using $PYTHON_VERSION, which is unsupported)."
+      puts-warn "We recommend upgrading by specifying the latest version ($LATEST_37)."
       echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+      else
+        echo "       Using supported version of Python 3.7 ($PYTHON_VERSION)"
+      fi
+    else
+      if [[ $PYTHON_VERSION =~ ^python-3.6 ]]; then
+        if [[ "$PYTHON_VERSION" != "$LATEST_36" ]]; then
+          puts-warn "The latest version of Python 3.6 is $LATEST_36 (you are using $PYTHON_VERSION, which is unsupported)."
+          puts-warn "We recommend upgrading by specifying the latest version ($LATEST_36)."
+          echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+        else
+          echo "       Using supported version of Python 3.6 ($PYTHON_VERSION)"
+        fi
+      else
+        puts-warn "Heroku supports runtime versions $LATEST_37, $LATEST_36 and $LATEST_2." 
+        puts-warn "You are using $PYTHON_VERSION, which is unsupported."
+        puts-warn "We recommend upgrading by specifying the default supported version ($LATEST_36)."
+        echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+      fi
     fi
   fi
 fi
@@ -49,6 +66,7 @@ if [ -f .heroku/python-version ]; then
   fi
 fi
 
+
 if [ ! "$SKIP_INSTALL" ]; then
     puts-step "Installing $PYTHON_VERSION"
 
@@ -71,6 +89,7 @@ if [ ! "$SKIP_INSTALL" ]; then
   hash -r
 fi
 
+
 # If Pip isn't up to date:
 if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
 

etc/ci-setup.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+sudo apt-get -qq update
+sudo apt-get install software-properties-common
+curl --fail --retry 3 --retry-delay 1 --connect-timeout 3 --max-time 30 https://cli-assets.heroku.com/install-ubuntu.sh | sh

hatchet.json

@@ -0,0 +1,5 @@
+{
+  "python": [
+    "heroku/python-getting-started"
+  ]
+}

hatchet.lock

@@ -0,0 +1,3 @@
+---
+- - "./repos/python/python-getting-started"
+  - f56b90499ec11e1c9576da2f8c7331300e189db3

spec/hatchet/python_spec.rb

@@ -0,0 +1,10 @@
+require_relative '../spec_helper'
+
+describe "Python!!!!!!!!!!!" do
+  it "🐍" do
+    Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app|
+      expect(app.output).to           match(/Installing pip/)
+      expect(app.run('python -V')).to match(/3.6.6/)
+    end
+  end
+end

spec/spec_helper.rb

@@ -0,0 +1,23 @@
+ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/heroku/heroku-buildpack-python.git'
+
+require 'rspec/core'
+require 'rspec/retry'
+require 'hatchet'
+
+require 'date'
+
+RSpec.configure do |config|
+  config.full_backtrace      = true
+  config.verbose_retry       = true # show retry status in spec process
+  config.default_retry_count = 2 if ENV['IS_RUNNING_ON_CI'] # retry all tests that fail again
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end
+
+if ENV['TRAVIS']
+  # Don't execute tests against "merge" commits
+  exit 0 if ENV['TRAVIS_PULL_REQUEST'] != 'false' && ENV['TRAVIS_BRANCH'] == 'master'
+end
+
+DEFAULT_STACK = 'heroku-16'

test/fixtures/airflow/requirements.txt

@@ -0,0 +1 @@
+apache-airflow==1.10

test/run

@@ -1,5 +1,12 @@
 #!/usr/bin/env bash
 
+testAirflow() {
+  export SLUGIFY_USES_TEXT_UNIDECODE="yes"
+  compile "airflow"
+  assertCaptured "apache-airflow==1.10"
+  assertCapturedSuccess
+}
+
 testPipenv() {
   compile "pipenv"
   assertCapturedSuccess
@@ -81,18 +88,9 @@ testPylibmc() {
 }
 
 testPython2() {
-  if [[ "$STACK" == "heroku-16" ]] || [[ "$STACK" == "cedar-14" ]]; then
     compile "python2"
     assertCaptured "python-2.7.15"
     assertCapturedSuccess
-  fi
-}
-
-testNoPython2() {
-  if [[ "$STACK" == "heroku-18" ]]; then
-    compile "python2"
-    assertCapturedError
-  fi
 }
 
 testPython3() {