From 9a9e972db0649adde53864e110c47fe680f5b8f3 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 00:06:28 -0500 Subject: [PATCH 01/13] Changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b74422..11babbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ # Master +- New pythons released: + CPython 3.8.1, 3.7.6, 3.6.10 and 3.9.0a2 + Beta Release: Pypy 2.7 and 3.6, version 7.2.0 + -------------------------------------------------------------------------------- # 162 (2019-12-06) From 514c4948918c28b84a68486a4b06856df255e261 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 00:08:04 -0500 Subject: [PATCH 02/13] Add new Python runtimes --- builds/runtimes/python-3.6.10 | 4 ++++ builds/runtimes/python-3.7.6 | 4 ++++ builds/runtimes/python-3.8.1 | 4 ++++ builds/runtimes/python3 | 8 +++----- 4 files changed, 15 insertions(+), 5 deletions(-) create mode 100755 builds/runtimes/python-3.6.10 create mode 100755 builds/runtimes/python-3.7.6 create mode 100755 builds/runtimes/python-3.8.1 diff --git a/builds/runtimes/python-3.6.10 b/builds/runtimes/python-3.6.10 new file mode 100755 index 0000000..ad41723 --- /dev/null +++ b/builds/runtimes/python-3.6.10 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python-3.7.6 b/builds/runtimes/python-3.7.6 new file mode 100755 index 0000000..ad41723 --- /dev/null +++ b/builds/runtimes/python-3.7.6 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python-3.8.1 b/builds/runtimes/python-3.8.1 new file mode 100755 index 0000000..ad41723 --- /dev/null +++ b/builds/runtimes/python-3.8.1 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python3 b/builds/runtimes/python3 index 1b6d152..6e7c517 100755 --- a/builds/runtimes/python3 +++ b/builds/runtimes/python3 @@ -16,6 +16,9 @@ python_version=${BASE^} # this gives us only the filename with version number version_number=$(echo "$python_version" | cut -d- -f2) # this returns just X.X.X dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz +echo "Building Python 3..." +echo "Pulling from source: ${dep_url}" + curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}" mv "${OUT_PREFIX}/${python_version}" src cd src @@ -31,9 +34,4 @@ find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm - # Remove spare / LOCATION=${OUT_PREFIX%?} -# Create links to SQLITE headers so Python can call them at runtime -mkdir -p ${OUT_PREFIX}/include ${OUT_PREFIX}/lib/x86_64-linux-gnu -cp /usr/include/sqlite3*.h ${OUT_PREFIX}/include -ln -fs $(realpath /usr/lib/x86_64-linux-gnu/libsqlite3.so) ${OUT_PREFIX}/lib/x86_64-linux-gnu/libsqlite3.so - ln $LOCATION/bin/python3 $LOCATION/bin/python From 8253ffa354658eece4b317bd27d245ac6cf572de Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 00:10:39 -0500 Subject: [PATCH 03/13] Default version updates --- bin/default_pythons | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bin/default_pythons b/bin/default_pythons index 16f044b..c37d9d8 100755 --- a/bin/default_pythons +++ b/bin/default_pythons @@ -1,11 +1,17 @@ #!/usr/bin/env bash -DEFAULT_PYTHON_VERSION="python-3.6.9" -LATEST_38="python-3.8.0" -LATEST_37="python-3.7.5" -LATEST_36="python-3.6.9" +DEFAULT_PYTHON_VERSION="python-3.6.10" +LATEST_38="python-3.8.1" +LATEST_37="python-3.7.6" +LATEST_36="python-3.6.10" LATEST_35="python-3.5.7" LATEST_34="python-3.4.10" LATEST_27="python-2.7.17" -export DEFAULT_PYTHON_VERSION LATEST_38 LATEST_37 LATEST_36 LATEST_35 LATEST_34 LATEST_27 +export DEFAULT_PYTHON_VERSION \ + LATEST_38 \ + LATEST_37 \ + LATEST_36 \ + LATEST_35 \ + LATEST_34 \ + LATEST_27 From f189df54151839a90973f361d787976ef28427ad Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 00:13:54 -0500 Subject: [PATCH 04/13] Warn about EOL on Python 2 install --- bin/steps/python | 10 +++++++++- test/run-versions | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/steps/python b/bin/steps/python index 2625b44..9046552 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -9,6 +9,10 @@ VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz" SECURITY_UPDATE="Python has released a security update! Please consider upgrading to" +ONLY_SUPPORTED_2_VERSION="Only the latest version of Python 2 is supported on the platform. Please consider upgrading to" + +PYTHON_2_EOL_UPDATE="Python 2 has reached it's community EOL. Upgrade your Python runtime to maintain a secure application as soon as possible." + # check if runtime exists if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then if [[ "$PYTHON_VERSION" == $PY38* ]]; then @@ -48,8 +52,12 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then fi if [[ "$PYTHON_VERSION" == $PY27* ]]; then # security update note + if [[ $(date "+%Y") > "2019" ]]; then + puts-warn "$PYTHON_2_EOL_UPDATE" + echo " Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq" + fi if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then - puts-warn "$SECURITY_UPDATE" "$LATEST_27" + puts-warn "$ONLY_SUPPORTED_2_VERSION" "$LATEST_27" echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" fi fi diff --git a/test/run-versions b/test/run-versions index 0698b7c..f9fbc66 100755 --- a/test/run-versions +++ b/test/run-versions @@ -18,6 +18,7 @@ testPython2() { echo $LATEST_27 > "runtime.txt" compile "python2" assertCaptured $LATEST_27 + assertCaptured "python-2-7-eol-faq" assertNotCaptured "security update" assertCaptured "Installing SQLite3" assertCapturedSuccess @@ -26,7 +27,8 @@ testPython2() { testPython2_warn() { compile "python2_warn" assertCaptured "python-2.7.15" - assertCaptured "security update!" + assertCaptured "python-2-7-eol-faq" + assertCaptured "Only the latest version" assertCaptured "Installing SQLite3" assertCapturedSuccess } From 6fc11b83703ad05c1807d44ce4427778d24b59c4 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 00:15:44 -0500 Subject: [PATCH 05/13] add up-to-date Pypy runtimes --- builds/runtimes/pypy2.7 | 23 +++++++++++++++++++++++ builds/runtimes/pypy2.7-7.2.0 | 4 ++++ builds/runtimes/pypy3.6 | 23 +++++++++++++++++++++++ builds/runtimes/pypy3.6-7.2.0 | 4 ++++ 4 files changed, 54 insertions(+) create mode 100755 builds/runtimes/pypy2.7 create mode 100755 builds/runtimes/pypy2.7-7.2.0 create mode 100755 builds/runtimes/pypy3.6 create mode 100755 builds/runtimes/pypy3.6-7.2.0 diff --git a/builds/runtimes/pypy2.7 b/builds/runtimes/pypy2.7 new file mode 100755 index 0000000..698aa17 --- /dev/null +++ b/builds/runtimes/pypy2.7 @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# fail hard +set -o pipefail +# fail harder +set -eu + +OUT_PREFIX=$1 + +dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 +dep_name=$(basename $BASH_SOURCE) # this is us +dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name +dep_package=${dep_name}-v${dep_version} # it's always "pypy2-…" +dep_dirname=${dep_package}-linux64 +dep_archive_name=${dep_dirname}.tar.bz2 +dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name} + +echo "Building PyPy…" +echo "${dep_url}" + +curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name + +ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python" diff --git a/builds/runtimes/pypy2.7-7.2.0 b/builds/runtimes/pypy2.7-7.2.0 new file mode 100755 index 0000000..40761e9 --- /dev/null +++ b/builds/runtimes/pypy2.7-7.2.0 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/pypy2.7 diff --git a/builds/runtimes/pypy3.6 b/builds/runtimes/pypy3.6 new file mode 100755 index 0000000..f24f806 --- /dev/null +++ b/builds/runtimes/pypy3.6 @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# fail hard +set -o pipefail +# fail harder +set -eu + +OUT_PREFIX=$1 + +dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 +dep_name=$(basename $BASH_SOURCE) # this is us +dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name +dep_package=${dep_name}${dep_version_prefix:-}-v${dep_version}${dep_version_suffix:-} +dep_dirname=${dep_package}-linux64 +dep_archive_name=${dep_dirname}.tar.bz2 +dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name} + +echo "Building PyPy3…" +echo "${dep_url}" + +curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name + +ln "$OUT_PREFIX/bin/pypy3" "$OUT_PREFIX/bin/python" diff --git a/builds/runtimes/pypy3.6-7.2.0 b/builds/runtimes/pypy3.6-7.2.0 new file mode 100755 index 0000000..e429be8 --- /dev/null +++ b/builds/runtimes/pypy3.6-7.2.0 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/pypy3.6 From 179f345f5b63d0ea4ec97729d913c3056d737d6f Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 00:16:20 -0500 Subject: [PATCH 06/13] add beta Pypy support --- bin/compile | 2 ++ bin/default_pythons | 6 +++++- bin/steps/python | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 9319463..3f8ce2a 100755 --- a/bin/compile +++ b/bin/compile @@ -58,6 +58,8 @@ PY36="python-3.6" PY35="python-3.5" PY34="python-3.4" PY27="python-2.7" +PYPY27="pypy2.7" +PYPY36="pypy3.6" # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)? DEFAULT_PYTHON_STACK="cedar-14" diff --git a/bin/default_pythons b/bin/default_pythons index c37d9d8..a648667 100755 --- a/bin/default_pythons +++ b/bin/default_pythons @@ -7,6 +7,8 @@ LATEST_36="python-3.6.10" LATEST_35="python-3.5.7" LATEST_34="python-3.4.10" LATEST_27="python-2.7.17" +PYPY_36="pypy3.6-7.2.0" +PYPY_27="pypy2.7-7.2.0" export DEFAULT_PYTHON_VERSION \ LATEST_38 \ @@ -14,4 +16,6 @@ export DEFAULT_PYTHON_VERSION \ LATEST_36 \ LATEST_35 \ LATEST_34 \ - LATEST_27 + LATEST_27 \ + PYPY_36 \ + PYPY_27 diff --git a/bin/steps/python b/bin/steps/python index 9046552..a2971f1 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -61,6 +61,20 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" fi fi + if [[ "$PYTHON_VERSION" == $PYPY27* ]]; then + # security update note + if [ "$PYTHON_VERSION" != "$PYPY_27" ]; then + puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_27}?" + echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" + fi + fi + if [[ "$PYTHON_VERSION" == $PYPY36* ]]; then + # security update note + if [ "$PYTHON_VERSION" != "$PYPY_36" ]; then + puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_36}?" + echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" + fi + fi else puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)." puts-warn "Aborting. More info: https://devcenter.heroku.com/articles/python-support" From 848c846a3d1e875b710d6c0aae065b6b96074029 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 00:17:22 -0500 Subject: [PATCH 07/13] Add tests for beta Pypy support --- test/fixtures/pypy2_7/requirements.txt | 1 + test/fixtures/pypy2_7/runtime.txt | 1 + test/fixtures/pypy3_6/requirements.txt | 1 + test/fixtures/pypy3_6/runtime.txt | 1 + test/run-versions | 14 ++++++++++++++ 5 files changed, 18 insertions(+) create mode 100644 test/fixtures/pypy2_7/requirements.txt create mode 100644 test/fixtures/pypy2_7/runtime.txt create mode 100644 test/fixtures/pypy3_6/requirements.txt create mode 100644 test/fixtures/pypy3_6/runtime.txt diff --git a/test/fixtures/pypy2_7/requirements.txt b/test/fixtures/pypy2_7/requirements.txt new file mode 100644 index 0000000..d3e4ba5 --- /dev/null +++ b/test/fixtures/pypy2_7/requirements.txt @@ -0,0 +1 @@ +django diff --git a/test/fixtures/pypy2_7/runtime.txt b/test/fixtures/pypy2_7/runtime.txt new file mode 100644 index 0000000..c502b68 --- /dev/null +++ b/test/fixtures/pypy2_7/runtime.txt @@ -0,0 +1 @@ +pypy2.7-7.2.0 diff --git a/test/fixtures/pypy3_6/requirements.txt b/test/fixtures/pypy3_6/requirements.txt new file mode 100644 index 0000000..d3e4ba5 --- /dev/null +++ b/test/fixtures/pypy3_6/requirements.txt @@ -0,0 +1 @@ +django diff --git a/test/fixtures/pypy3_6/runtime.txt b/test/fixtures/pypy3_6/runtime.txt new file mode 100644 index 0000000..e1af38f --- /dev/null +++ b/test/fixtures/pypy3_6/runtime.txt @@ -0,0 +1 @@ +pypy3.6-7.2.0 diff --git a/test/run-versions b/test/run-versions index f9fbc66..fcfaf94 100755 --- a/test/run-versions +++ b/test/run-versions @@ -168,6 +168,20 @@ testPython3_8_fail() { assertCapturedError } +testPypy3_6() { + compile "pypy3_6" + assertCaptured "Installing Pypy 3.6" + assertCaptured "$PYPY_36" + assertCapturedSuccess +} + +testPypy2_7() { + compile "pypy2_7" + assertCaptured "Installing Pypy 2.7" + assertCaptured "$PYPY_27" + assertCapturedSuccess +} + pushd $(dirname 0) >/dev/null popd >/dev/null From 0e1ac6217b2a0358ee16facb67ca77751b1f5753 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 01:22:30 -0500 Subject: [PATCH 08/13] correct test expectations --- test/run-versions | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/run-versions b/test/run-versions index fcfaf94..ba24d87 100755 --- a/test/run-versions +++ b/test/run-versions @@ -18,7 +18,11 @@ testPython2() { echo $LATEST_27 > "runtime.txt" compile "python2" assertCaptured $LATEST_27 - assertCaptured "python-2-7-eol-faq" + if [[ $(date "+%Y") > "2019" ]]; then + assertCaptured "python-2-7-eol-faq"; + else + assertNotCaptured "python-2-7-eol-faq"; + fi assertNotCaptured "security update" assertCaptured "Installing SQLite3" assertCapturedSuccess @@ -27,7 +31,11 @@ testPython2() { testPython2_warn() { compile "python2_warn" assertCaptured "python-2.7.15" - assertCaptured "python-2-7-eol-faq" + if [[ $(date "+%Y") > "2019" ]]; then + assertCaptured "python-2-7-eol-faq"; + else + assertNotCaptured "python-2-7-eol-faq"; + fi assertCaptured "Only the latest version" assertCaptured "Installing SQLite3" assertCapturedSuccess @@ -170,14 +178,14 @@ testPython3_8_fail() { testPypy3_6() { compile "pypy3_6" - assertCaptured "Installing Pypy 3.6" + assertCaptured "Installing pypy" assertCaptured "$PYPY_36" assertCapturedSuccess } testPypy2_7() { compile "pypy2_7" - assertCaptured "Installing Pypy 2.7" + assertCaptured "Installing pypy" assertCaptured "$PYPY_27" assertCapturedSuccess } From 2942fc8e4a1c336e800ce6288d72c51e0413ab7c Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 01:28:06 -0500 Subject: [PATCH 09/13] Correct Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11babbc..1b1d08b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ # Master - New pythons released: - CPython 3.8.1, 3.7.6, 3.6.10 and 3.9.0a2 + Python 3.8.1, 3.7.6, 3.6.10 (CPython) Beta Release: Pypy 2.7 and 3.6, version 7.2.0 -------------------------------------------------------------------------------- From 7d743e7998bc83dda178d8e4a1445c61b130b28e Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 09:57:07 -0500 Subject: [PATCH 10/13] Don't test for pypy on cedar 14 --- test/run-versions | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/run-versions b/test/run-versions index ba24d87..06f1c3d 100755 --- a/test/run-versions +++ b/test/run-versions @@ -178,6 +178,9 @@ testPython3_8_fail() { testPypy3_6() { compile "pypy3_6" + if [[ $STACK = "cedar-14" ]]; then + assertCapturedError + else assertCaptured "Installing pypy" assertCaptured "$PYPY_36" assertCapturedSuccess @@ -185,6 +188,9 @@ testPypy3_6() { testPypy2_7() { compile "pypy2_7" + if [[ $STACK = "cedar-14" ]]; then + assertCapturedError + else assertCaptured "Installing pypy" assertCaptured "$PYPY_27" assertCapturedSuccess From 04199212271f4a3be79d3d2ace075704d5ad880d Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 10:56:14 -0500 Subject: [PATCH 11/13] bash is not python, so use bash block closers --- test/run-versions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/run-versions b/test/run-versions index 06f1c3d..7076654 100755 --- a/test/run-versions +++ b/test/run-versions @@ -180,7 +180,7 @@ testPypy3_6() { compile "pypy3_6" if [[ $STACK = "cedar-14" ]]; then assertCapturedError - else + fi assertCaptured "Installing pypy" assertCaptured "$PYPY_36" assertCapturedSuccess @@ -190,7 +190,7 @@ testPypy2_7() { compile "pypy2_7" if [[ $STACK = "cedar-14" ]]; then assertCapturedError - else + fi assertCaptured "Installing pypy" assertCaptured "$PYPY_27" assertCapturedSuccess From 0057d19082822d3cd0cfda1f3f282ae4df97ab27 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 12:28:31 -0500 Subject: [PATCH 12/13] Typo in bash if-else block --- test/run-versions | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/run-versions b/test/run-versions index 7076654..c207be1 100755 --- a/test/run-versions +++ b/test/run-versions @@ -180,20 +180,22 @@ testPypy3_6() { compile "pypy3_6" if [[ $STACK = "cedar-14" ]]; then assertCapturedError + else + assertCaptured "Installing pypy" + assertCaptured "$PYPY_36" + assertCapturedSuccess fi - assertCaptured "Installing pypy" - assertCaptured "$PYPY_36" - assertCapturedSuccess } testPypy2_7() { compile "pypy2_7" if [[ $STACK = "cedar-14" ]]; then assertCapturedError + else + assertCaptured "Installing pypy" + assertCaptured "$PYPY_27" + assertCapturedSuccess fi - assertCaptured "Installing pypy" - assertCaptured "$PYPY_27" - assertCapturedSuccess } pushd $(dirname 0) >/dev/null From ec57979bf8b917ecded8180a97f8c7bc3132649e Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Mon, 23 Dec 2019 13:14:06 -0500 Subject: [PATCH 13/13] comply with shellcheck, use -gt instead --- bin/steps/python | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/steps/python b/bin/steps/python index a2971f1..8790f61 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -52,7 +52,7 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then fi if [[ "$PYTHON_VERSION" == $PY27* ]]; then # security update note - if [[ $(date "+%Y") > "2019" ]]; then + if [[ "$(date "+%Y")" -gt "2019" ]]; then puts-warn "$PYTHON_2_EOL_UPDATE" echo " Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq" fi