Set Codeowners to languages (#953)

* Set languages as codeowners

* Update changelog

.github/CODEOWNERS

@@ -1 +1 @@
-* @heroku/languages
+* @heroku/languages

.gitignore

@@ -10,3 +10,5 @@ buildpack/*
 
 builds/dockerenv.staging*
 builds/dockerenv.production
+
+test/scratch

.travis.yml

@@ -10,11 +10,11 @@ before_script:
 
 script:
   - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
-  - docker run --rm -e "STACK=cedar-14" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-cedar-14 bash $TESTFOLDER
+  - docker run --rm -e "STACK=cedar-14" travis-build-cedar-14 bash $TESTFOLDER
   - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-  - docker run --rm -e "STACK=heroku-16" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-16 bash $TESTFOLDER
+  - docker run --rm -e "STACK=heroku-16" travis-build-heroku-16 bash $TESTFOLDER
   - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-  - docker run --rm -e "STACK=heroku-18" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-18 bash $TESTFOLDER
+  - docker run --rm -e "STACK=heroku-18" travis-build-heroku-18 bash $TESTFOLDER
 
 jobs:
   include:

CHANGELOG.md

@@ -2,11 +2,40 @@
 
 # Master
 
-- Update requirements.txt builds to use Pip 20.0.2
-- Download get-pip.py to tmpdir instead of root dir
+- Set Code Owners to @heroku/langauges
+- Bugfix: Caching on subsequent redeploys
+- Update tests to support latest version of Python
 
 --------------------------------------------------------------------------------
 
+# 168 (2020-04-06)
+
+- Doc: Update Readme with version numbers
+- update Code Owners to include the Heroku Buildpack Maintainers team
+- Deprecation warning: BUILD_WITH_GEO_LIBRARIES is now deprecated. See warning for details.
+- Clean up build log output
+- Update Python versions in README to match docs
+- Django version detection fixed, link updated
+
+# 167 (2020-03-26)
+
+- Add failcase for cache busting
+- Bugfix: Clearing pip dependencies
+
+# 166 (2020-03-05)
+
+- Correct ftp to https in vendored file
+- Warn for Django 1.11 approaching EOL, provide link to roadmap
+
+# 165 (2020-02-27)
+
+- Python 3.8.2 now available.
+
+# 164 (2020-02-20)
+
+- Update requirements.txt builds to use Pip 20.0.2
+- Download get-pip.py to tmpdir instead of root dir
+
 # 163 (2019-12-23)
 
 - New pythons released:

README.md

@@ -61,8 +61,9 @@ Specify a Python Runtime
 
 Supported runtime options include:
 
-- `python-3.7.4`
-- `python-3.6.9`
+- `python-3.8.2`
+- `python-3.7.6`
+- `python-3.6.10`
 - `python-2.7.17`
 
 ## Tests

bin/compile

@@ -175,7 +175,7 @@ cd "$BUILD_DIR"
 # - Once the build is complete, `~/.heroku/{known-paths}` is copied back into the cache.
 
 # Create the cache directory, if it doesn't exist.
-mkdir -p "$CACHE_DIR"
+mkdir -p "$CACHE_DIR/.heroku"
 
 # Restore old artifacts from the cache.
 mkdir -p .heroku
@@ -259,17 +259,8 @@ mtime "python.install.time" "${start}"
 # shellcheck source=bin/steps/pipenv
 source "$BIN_DIR/steps/pipenv"
 
-# Uninstall removed dependencies with Pip.
-# The buildpack will automatically remove any declared dependencies (in requirements.txt)
-# that were explicitly removed. This machinery is a bit complex, but it is not complicated.
-(( start=$(nowms) ))
-# shellcheck source=bin/steps/pip-uninstall
-source "$BIN_DIR/steps/pip-uninstall"
-mtime "pip.uninstall.time" "${start}"
-
 # If no requirements.txt file given, assume `setup.py develop` is intended.
 # This allows for people to ship a setup.py application to Heroku
-# (which is rare, but I vouch that it should work!)
 
 if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
   echo "-e ." > requirements.txt

bin/default_pythons

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 DEFAULT_PYTHON_VERSION="python-3.6.10"
-LATEST_38="python-3.8.1"
+LATEST_38="python-3.8.2"
 LATEST_37="python-3.7.6"
 LATEST_36="python-3.6.10"
 LATEST_35="python-3.5.7"

bin/steps/README.MD

@@ -0,0 +1,28 @@
+# Python Buildpack Install Steps
+
+TODO: Add context on Python install steps, such as why symlinking vs copying
+
+## Installing the Pip tool
+
+The Python Buildpack uses a tool called `get-pip` to install the pip tool. This
+is done in the `python` script.
+
+This is in part because Python historically did not come with pip by default.
+
+## Installing Python packages using Pip
+
+### Convention: Use `python` process to invoke Pip
+
+We don't use this convention (yet) but this is an upcoming change being considered.
+
+This is a bigger concern on Windows than it is in Linux environments, but an
+emerging convention in the Python community is to invoke pip using:
+
+```
+python3 -m pip [options]
+```
+
+Invoking pip this way ensures correct location - python knows where these
+packages are stored because it put them there (defaults to Python's pathing info).
+
+All normal command line options are available using this method.

bin/steps/gdal

@@ -22,6 +22,10 @@ source "$BIN_DIR/utils"
 if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
 
   if [ ! -f ".heroku/vendor/bin/gdalserver" ]; then
+
+    puts-warn "The vendored GDAL package in the Heroku Python Buildpack now deprecated."
+    puts-warn "To enable GDAL use an alternative buildpack is available here - https://github.com/heroku/heroku-geo-buildpack"
+
     echo "-----> Noticed GDAL. Bootstrapping gdal."
     mkdir -p .heroku/vendor
     # Download and extract cryptography into target vendor directory.

bin/steps/geo-libs

@@ -24,6 +24,9 @@ source "$BIN_DIR/utils"
 if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
   mcount "buildvar.BUILD_WITH_GEO_LIBRARIES"
 
+  puts-warn "The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated."
+  puts-warn "An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
+
   if [ ! -f ".heroku/vendor/bin/proj" ]; then
     echo "-----> Bootstrapping gdal, geos, proj."
     mkdir -p .heroku/vendor

bin/steps/pip-install

@@ -38,9 +38,16 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
         mcount "failure.none-version"
     fi
 
+    if grep -qi '^django==1.*' requirements.txt; then
+        puts-warn "Your Django version is nearing the end of its community support."
+        puts-warn "Upgrade to continue to receive security updates and for the best experience with Django."
+        puts-warn "For more information, check out https://www.djangoproject.com/download/#supported-versions"
+    fi
+
     if [ ! -f "$BUILD_DIR/.heroku/python/bin/pip" ]; then
         exit 1
     fi
+
     /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
     PIP_STATUS="${PIPESTATUS[0]}"
     set -e
@@ -52,12 +59,9 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
         exit 1
     fi
 
-    # Smart Requirements handling
     cp requirements.txt .heroku/python/requirements-declared.txt
     /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
 
-    echo
-
     # Install test dependencies, for CI.
     if [ "$INSTALL_TEST" ]; then
         if [[ -f "$1/requirements-test.txt" ]]; then

bin/steps/python

@@ -86,14 +86,6 @@ if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
     rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
 fi
 
-# need to clear the cache for first time installing SQLite3,
-# since the version is changing and could lead to runtime errors
-# with compiled extensions.
-if [ -d .heroku/python ] && [ ! -f .heroku/python-sqlite3-version ] && python_sqlite3_check "$PYTHON_VERSION"; then
-  puts-step "Need to update SQLite3, clearing cache"
-  rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
-fi
-
 if [ -f .heroku/python-version ]; then
   if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then
       puts-step "Found $(cat .heroku/python-version), removing"
@@ -103,6 +95,24 @@ if [ -f .heroku/python-version ]; then
   fi
 fi
 
+# Check if we should reinstall python dependencies
+if [[ ! -f "$CACHE_DIR/.heroku/requirements.txt" ]]; then
+  # IF there's no cached dependencies, update cached version of requirements.txt
+  # This should only run for new apps and first deploys after this update
+  cp -R "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt"
+else
+  # IF there IS a cached directory, check for differences with the new one
+  if ! diff "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt" &> /dev/null; then
+    puts-step "Requirements file has been changed, clearing cached dependencies"
+    # if there are any differences, clear the Python cache
+    # Installing Python over again does not take noticably more time
+    cp -R "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt"
+    rm -rf .heroku/python
+    unset SKIP_INSTALL
+  else
+    puts-step "No change in requirements detected, installing from cache"
+  fi
+fi
 
 if [ ! "$SKIP_INSTALL" ]; then
     puts-step "Installing $PYTHON_VERSION"
@@ -137,7 +147,7 @@ if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then
     exit 1
 fi
 
-# If Pip isn't up to date:
+# If a new Python has been installed or Pip isn't up to date:
 if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
 
   puts-step "Installing pip"

bin/steps/sqlite3

@@ -67,7 +67,6 @@ buildpack_sqlite3_install() {
   puts-step "Installing SQLite3"
 
   if sqlite3_install "$BUILD_DIR/.heroku/python" ; then
-    echo "Sqlite3 successfully installed."
     mcount "success.python.sqlite3"
   else
     echo "Sqlite3 failed to install."

builds/libraries/vendor/libffi

@@ -15,11 +15,11 @@ hash -r
 
 echo "Building libffi…"
 
-SOURCE_TARBALL='ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz'
+SOURCE_TARBALL='https://github.com/libffi/libffi/archive/v3.1.tar.gz'
 
 curl -L $SOURCE_TARBALL | tar x
 
-cd libffi-3.1
+cd v3.1
 ./configure --prefix=$OUT_PREFIX --disable-static &&
 make
 make install

builds/runtimes/python-3.8.2

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.3

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

hatchet.lock

@@ -1,3 +1,3 @@
 ---
 - - "./repos/python/python-getting-started"
-  - f56b90499ec11e1c9576da2f8c7331300e189db3
+  - 443a90c58be6881583cd7ef628e3869e3c30bb98

requirements.txt

@@ -1,3 +1,3 @@
 docopt==0.6.2
-bob-builder
+bob-builder==0.0.17
 boto==2.48.0

spec/hatchet/python_spec.rb

@@ -1,10 +1,42 @@
 require_relative '../spec_helper'
 
-describe "Python!!!!!!!!!!!" do
+describe "Default Python Deploy" do
   it "🐍" do
     Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app|
+
+      # What should happen on first deploy
       expect(app.output).to           match(/Installing pip/)
-      expect(app.run('python -V')).to match(/3.6.10/)
+
+      # What should not happen
+      expect(app.output).to_not match("Requirements file has been changed, updating cache")
+      expect(app.output).to_not match("No change in requirements detected, installing from cache")
+      expect(app.output).to_not match("No such file or directory")
+      expect(app.output).to_not match("cp: cannot create regular file")
+
+      # Redeploy with changed requirements file
+      run!(%Q{echo "" >> requirements.txt})
+      run!(%Q{echo "flask" >> requirements.txt})
+      run!(%Q{git add . ; git commit --allow-empty -m next})
+      app.push!
+
+      # Check for the cache to have cleared
+      expect(app.output).to match("Requirements file has been changed, updating cache")
+
+      # What should not happen when the requirements file is changed
+      expect(app.output).to_not match("No dependencies found, preparing to install")
+      expect(app.output).to_not match("No change in requirements detected, installing from cache")
+
+      run!(%Q{git commit --allow-empty -m next})
+      app.push!
+
+      # With no changes on redeploy, the cache should
+      expect(app.output).to match("No change in requirements detected, installing from cache")
+
+      # With no changes on redeploy, the cache should not
+      expect(app.output).to_not match("Requirements file has been changed, updating cache")
+      expect(app.output).to_not match("No dependencies found, preparing to install")
+
+      expect(app.run('python -V')).to match(/3.7.6/)
     end
   end
 end

spec/spec_helper.rb

@@ -20,4 +20,10 @@ if ENV['TRAVIS']
   exit 0 if ENV['TRAVIS_PULL_REQUEST'] != 'false' && ENV['TRAVIS_BRANCH'] == 'master'
 end
 
-DEFAULT_STACK = 'heroku-16'
+DEFAULT_STACK = 'heroku-18'
+
+def run!(cmd)
+  out = `#{cmd}`
+  raise "Error running command #{cmd} with output: #{out}" unless $?.success?
+  return out
+end

test/fixtures/not-old-django/requirements.txt

@@ -0,0 +1,2 @@
+pwned-passwords-django==1.4
+Django==2.1

test/fixtures/old-django/requirements.txt

@@ -0,0 +1 @@
+Django==1.11

test/fixtures/python3_8/runtime.txt

@@ -1 +1 @@
-python-3.8.0
+python-3.8.2

test/fixtures/python3_8_warn/requirements.txt

@@ -0,0 +1 @@
+requests

test/fixtures/python3_8_warn/runtime.txt

@@ -0,0 +1 @@
+python-3.8.0

test/run-deps

@@ -23,6 +23,14 @@ testGEOS() {
   assertCapturedSuccess
 }
 
+testGEOSDeprecation() {
+  export BUILD_WITH_GEO_LIBRARIES=1
+  compile "geos"
+  assertCaptured " !     The GDAL, GEOS and PROJ binaries and BUILD_WITH_GEO_LIBRARIES functonality are now deprecated.
+ !     An alternative buildpack to enable GDAL, GEOS and PROJ use is available here - https://github.com/heroku/heroku-geo-buildpack"
+  assertCapturedSuccess
+}
+
 testNLTK() {
   # NOTE: This is a RuntimeWarning emitted by Python 3's runpy.py script
   # which is what is used when you call `python -m <module>`. This is due to
@@ -48,7 +56,6 @@ testPysqlite() {
 
 testSqliteInstall() {
   compile "pythonDefault"
-  assertCaptured "Sqlite3 successfully installed."
   assertNotCaptured "Sqlite3 failed to install."
   assertCapturedSuccess
 }

test/run-features

@@ -74,6 +74,18 @@ testNoRequirements() {
   assertCapturedError
 }
 
+testWarnOldDjango() {
+  compile "old-django"
+  assertCaptured "Your Django version is nearing the end of its community support."
+  assertCapturedSuccess
+}
+
+testDontWarnOldDjango() {
+  compile "not-old-django"
+  assertNotCaptured "Your Django version is nearing the end of its community support."
+  assertCapturedSuccess
+}
+
 pushd $(dirname 0) >/dev/null
 popd >/dev/null
 

test/run-versions

@@ -157,6 +157,18 @@ testPython3_7_fail() {
   assertCapturedError
 }
 
+testPython3_7_warn() {
+  compile "python3_8_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "python-3.8.0"
+    assertCaptured "security update!"
+    assertCaptured "Installing SQLite3"
+    assertCapturedSuccess
+  fi
+}
+
 testPython3_8() {
   updateVersion "python3_8" $LATEST_38
   compile "python3_8"