Merge pull request #920 from heroku/python/382

Build and release Python 3.8.2

CHANGELOG.md

@@ -2,10 +2,26 @@
 
 # Master
 
-- Bug fix: fragile sqlite3 install
+- Python 3.8.2 now available.
+
 
 --------------------------------------------------------------------------------
 
+# 163 (2020-02-20)
+
+- Update requirements.txt builds to use Pip 20.0.2
+- Download get-pip.py to tmpdir instead of root dir
+
+# 163 (2019-12-23)
+
+- New pythons released:
+  Python 3.8.1, 3.7.6, 3.6.10 (CPython)
+  Beta Release: Pypy 2.7 and 3.6, version 7.2.0
+
+# 162 (2019-12-06)
+
+- Bug fix: fragile sqlite3 install
+
 # 161 (2019-12-2)
 
 - Bug fix: Sqlite3 version bump

bin/compile

@@ -58,11 +58,31 @@ PY36="python-3.6"
 PY35="python-3.5"
 PY34="python-3.4"
 PY27="python-2.7"
+PYPY27="pypy2.7"
+PYPY36="pypy3.6"
 
 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
 # If pip doesn't match this version (the version we install), run the installer.
-PIP_UPDATE="9.0.2"
+PIP_UPDATE="20.0.2"
+
+for file in "$BUILD_DIR/runtime.txt" "$CACHE_DIR/.heroku/python-version" ; do
+    [ -f "$file" ] || continue
+
+    version=$(tr -d '[:space:]' < "$file")
+
+    case "$version" in "$PY34"*)
+        # Python 3.4 support was dropped in pip >= 19.2.
+        PIP_UPDATE="19.1.1"
+        break
+        ;;
+    esac
+done
+
+if [[ -f "$BUILD_DIR/Pipfile" ]]; then
+    # Do not force pipenv users to re-install pipenv locally.
+    PIP_UPDATE="9.0.2"
+fi
 
 export DEFAULT_PYTHON_STACK PIP_UPDATE
 export PY37 PY36 PY35 PY27 PY34

bin/default_pythons

@@ -1,11 +1,21 @@
 #!/usr/bin/env bash
 
-DEFAULT_PYTHON_VERSION="python-3.6.9"
-LATEST_38="python-3.8.0"
-LATEST_37="python-3.7.5"
-LATEST_36="python-3.6.9"
+DEFAULT_PYTHON_VERSION="python-3.6.10"
+LATEST_38="python-3.8.2"
+LATEST_37="python-3.7.6"
+LATEST_36="python-3.6.10"
 LATEST_35="python-3.5.7"
 LATEST_34="python-3.4.10"
 LATEST_27="python-2.7.17"
+PYPY_36="pypy3.6-7.2.0"
+PYPY_27="pypy2.7-7.2.0"
 
-export DEFAULT_PYTHON_VERSION LATEST_38 LATEST_37 LATEST_36 LATEST_35 LATEST_34 LATEST_27
+export DEFAULT_PYTHON_VERSION \
+    LATEST_38 \
+    LATEST_37 \
+    LATEST_36 \
+    LATEST_35 \
+    LATEST_34 \
+    LATEST_27 \
+    PYPY_36 \
+    PYPY_27

bin/steps/python

@@ -9,6 +9,10 @@ VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"
 
 SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
 
+ONLY_SUPPORTED_2_VERSION="Only the latest version of Python 2 is supported on the platform. Please consider upgrading to"
+
+PYTHON_2_EOL_UPDATE="Python 2 has reached it's community EOL. Upgrade your Python runtime to maintain a secure application as soon as possible."
+
 # check if runtime exists
 if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
   if [[ "$PYTHON_VERSION" == $PY38* ]]; then
@@ -48,8 +52,26 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
   fi
   if [[ "$PYTHON_VERSION" == $PY27* ]]; then
     # security update note
+    if [[ "$(date "+%Y")" -gt "2019" ]]; then
+      puts-warn "$PYTHON_2_EOL_UPDATE"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq"
+    fi
     if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then
-      puts-warn "$SECURITY_UPDATE" "$LATEST_27"
+      puts-warn "$ONLY_SUPPORTED_2_VERSION" "$LATEST_27"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PYPY27* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$PYPY_27" ]; then
+      puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_27}?"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PYPY36* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$PYPY_36" ]; then
+      puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_36}?"
       echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
     fi
   fi
@@ -106,9 +128,10 @@ fi
 
 # Heroku uses the get-pip utility maintained by the Python community to vendor Pip.
 # https://github.com/pypa/get-pip
-GETPIP="https://lang-python.s3.amazonaws.com/etc/get-pip.py"
+GETPIP_URL="https://lang-python.s3.amazonaws.com/etc/get-pip.py"
+GETPIP_PY="${TMPDIR:-/tmp}/get-pip.py"
 
-if ! curl -s "${GETPIP}" -o "$ROOT_DIR/get-pip.py" &> /dev/null; then
+if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then
     mcount "failure.python.get-pip"
     echo "Failed to pull down get-pip"
     exit 1
@@ -123,7 +146,7 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then
   rm -fr /app/.heroku/python/lib/python*/site-packages/pip-*
   rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-*
 
-  /app/.heroku/python/bin/python "$ROOT_DIR/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null
+  /app/.heroku/python/bin/python "$GETPIP_PY" pip=="$PIP_UPDATE" &> /dev/null
   /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null
 fi
 

builds/runtimes/pypy2.7

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# fail hard
+set -o pipefail
+# fail harder
+set -eu
+
+OUT_PREFIX=$1
+
+dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
+dep_name=$(basename $BASH_SOURCE) # this is us
+dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name
+dep_package=${dep_name}-v${dep_version} # it's always "pypy2-…"
+dep_dirname=${dep_package}-linux64
+dep_archive_name=${dep_dirname}.tar.bz2
+dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+
+echo "Building PyPy…"
+echo "${dep_url}"
+
+curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+
+ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python"

builds/runtimes/pypy2.7-7.2.0

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy2.7

builds/runtimes/pypy3.6

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# fail hard
+set -o pipefail
+# fail harder
+set -eu
+
+OUT_PREFIX=$1
+
+dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1
+dep_name=$(basename $BASH_SOURCE) # this is us
+dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name
+dep_package=${dep_name}${dep_version_prefix:-}-v${dep_version}${dep_version_suffix:-}
+dep_dirname=${dep_package}-linux64
+dep_archive_name=${dep_dirname}.tar.bz2
+dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name}
+
+echo "Building PyPy3…"
+echo "${dep_url}"
+
+curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name
+
+ln "$OUT_PREFIX/bin/pypy3" "$OUT_PREFIX/bin/python"

builds/runtimes/pypy3.6-7.2.0

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/pypy3.6

builds/runtimes/python-3.6.10

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.7.6

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.1

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.2

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python-3.8.3

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

builds/runtimes/python3

@@ -16,6 +16,9 @@ python_version=${BASE^} # this gives us only the filename with version number
 version_number=$(echo "$python_version" | cut -d- -f2) # this returns just X.X.X
 dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz
 
+echo "Building Python 3..."
+echo "Pulling from source: ${dep_url}"
+
 curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}"
 mv "${OUT_PREFIX}/${python_version}" src
 cd src
@@ -31,9 +34,4 @@ find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -
 # Remove spare /
 LOCATION=${OUT_PREFIX%?}
 
-# Create links to SQLITE headers so Python can call them at runtime
-mkdir -p ${OUT_PREFIX}/include ${OUT_PREFIX}/lib/x86_64-linux-gnu
-cp /usr/include/sqlite3*.h ${OUT_PREFIX}/include
-ln -fs $(realpath /usr/lib/x86_64-linux-gnu/libsqlite3.so) ${OUT_PREFIX}/lib/x86_64-linux-gnu/libsqlite3.so
-
 ln $LOCATION/bin/python3 $LOCATION/bin/python

requirements.txt

@@ -1,3 +1,3 @@
 docopt==0.6.2
-bob-builder
+bob-builder==0.0.17
 boto==2.48.0

spec/hatchet/python_spec.rb

@@ -4,7 +4,7 @@ describe "Python!!!!!!!!!!!" do
   it "🐍" do
     Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app|
       expect(app.output).to           match(/Installing pip/)
-      expect(app.run('python -V')).to match(/3.6.9/)
+      expect(app.run('python -V')).to match(/3.6.10/)
     end
   end
 end

test/fixtures/pypy2_7/requirements.txt

@@ -0,0 +1 @@
+django

test/fixtures/pypy2_7/runtime.txt

@@ -0,0 +1 @@
+pypy2.7-7.2.0

test/fixtures/pypy3_6/requirements.txt

@@ -0,0 +1 @@
+django

test/fixtures/pypy3_6/runtime.txt

@@ -0,0 +1 @@
+pypy3.6-7.2.0

test/fixtures/python3_8/runtime.txt

@@ -1 +1 @@
-python-3.8.0
+python-3.8.2

test/fixtures/python3_8_warn/requirements.txt

@@ -0,0 +1 @@
+requests

test/fixtures/python3_8_warn/runtime.txt

@@ -0,0 +1 @@
+python-3.8.0

test/run-versions

@@ -18,6 +18,11 @@ testPython2() {
     echo $LATEST_27 > "runtime.txt"
     compile "python2"
     assertCaptured $LATEST_27
+    if [[ $(date "+%Y") > "2019" ]]; then
+      assertCaptured "python-2-7-eol-faq";
+    else
+      assertNotCaptured "python-2-7-eol-faq";
+    fi
     assertNotCaptured "security update"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
@@ -26,7 +31,12 @@ testPython2() {
 testPython2_warn() {
     compile "python2_warn"
     assertCaptured "python-2.7.15"
-    assertCaptured "security update!"
+    if [[ $(date "+%Y") > "2019" ]]; then
+      assertCaptured "python-2-7-eol-faq";
+    else
+      assertNotCaptured "python-2-7-eol-faq";
+    fi
+    assertCaptured "Only the latest version"
     assertCaptured "Installing SQLite3"
     assertCapturedSuccess
 }
@@ -147,6 +157,18 @@ testPython3_7_fail() {
   assertCapturedError
 }
 
+testPython3_7_warn() {
+  compile "python3_8_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "python-3.8.0"
+    assertCaptured "security update!"
+    assertCaptured "Installing SQLite3"
+    assertCapturedSuccess
+  fi
+}
+
 testPython3_8() {
   updateVersion "python3_8" $LATEST_38
   compile "python3_8"
@@ -166,6 +188,28 @@ testPython3_8_fail() {
   assertCapturedError
 }
 
+testPypy3_6() {
+  compile "pypy3_6"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "Installing pypy"
+    assertCaptured "$PYPY_36"
+    assertCapturedSuccess
+  fi
+}
+
+testPypy2_7() {
+  compile "pypy2_7"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "Installing pypy"
+    assertCaptured "$PYPY_27"
+    assertCapturedSuccess
+  fi
+}
+
 pushd $(dirname 0) >/dev/null
 popd >/dev/null
 

vendor/pip-pop/pip-diff

@@ -12,9 +12,24 @@ Options:
 """
 import os
 from docopt import docopt
-from pip.req import parse_requirements
-from pip.index import PackageFinder
-from pip._vendor.requests import session
+
+try:  # pip >= 10
+    from pip._internal.req import parse_requirements
+    from pip._internal.download import PipSession as session
+
+    def PackageFinder(find_links, index_urls, session=None):
+        from pip._internal.index import PackageFinder
+        from pip._internal.models.search_scope import SearchScope
+        from pip._internal.models.selection_prefs import SelectionPreferences
+
+        search_scope = SearchScope.create(find_links, index_urls)
+        selection_prefs = SelectionPreferences(allow_yanked=False)
+        return PackageFinder.create(search_scope, selection_prefs, session=session)
+
+except ImportError:  # pip <= 9.0.3
+    from pip.req import parse_requirements
+    from pip.index import PackageFinder
+    from pip._vendor.requests import session
 
 requests = session()
 

vendor/pip-pop/pip-grep

@@ -10,9 +10,25 @@ Options:
 import os
 import sys
 from docopt import docopt
-from pip.req import parse_requirements
-from pip.index import PackageFinder
-from pip._vendor.requests import session
+
+try:  # pip >= 10
+    from pip._internal.req import parse_requirements
+    from pip._internal.download import PipSession as session
+
+    def PackageFinder(find_links, index_urls, session=None):
+        from pip._internal.index import PackageFinder
+        from pip._internal.models.search_scope import SearchScope
+        from pip._internal.models.selection_prefs import SelectionPreferences
+
+        search_scope = SearchScope.create(find_links, index_urls)
+        selection_prefs = SelectionPreferences(allow_yanked=False)
+        return PackageFinder.create(search_scope, selection_prefs, session=session)
+
+except ImportError:  # pip <= 9.0.3
+    from pip.req import parse_requirements
+    from pip.index import PackageFinder
+    from pip._vendor.requests import session
+
 
 requests = session()