diff --git a/.travis.yml b/.travis.yml index 703f4a1..102e16e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,11 +28,8 @@ jobs: env: matrix: - TESTFOLDER=test/run-deps - - TESTFOLDER=test/run-deps USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging - TESTFOLDER=test/run-versions - - TESTFOLDER=test/run-versions USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging - TESTFOLDER=test/run-features - - TESTFOLDER=test/run-features USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging global: - HATCHET_RETRIES=3 - IS_RUNNING_ON_CI=true diff --git a/CHANGELOG.md b/CHANGELOG.md index 2132849..1b1d08b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,20 @@ # Master -- Sqlite fix: - - Update Python3 and Python2 base formula - - Update Python formulas 3.6.x, 3.7.x, 3.5.x, and 2.7.17-2.7.9 -- Test staged binaries on Travis +- New pythons released: + Python 3.8.1, 3.7.6, 3.6.10 (CPython) + Beta Release: Pypy 2.7 and 3.6, version 7.2.0 -------------------------------------------------------------------------------- +# 162 (2019-12-06) + +- Bug fix: fragile sqlite3 install + +# 161 (2019-12-2) + +- Bug fix: Sqlite3 version bump + # 160 (2019-10-23) - Bugfix: Pipenv no longer installs twice in CI diff --git a/bin/compile b/bin/compile index db2d838..b5d6ed6 100755 --- a/bin/compile +++ b/bin/compile @@ -58,6 +58,8 @@ PY36="python-3.6" PY35="python-3.5" PY34="python-3.4" PY27="python-2.7" +PYPY27="pypy2.7" +PYPY36="pypy3.6" # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)? DEFAULT_PYTHON_STACK="cedar-14" diff --git a/bin/default_pythons b/bin/default_pythons index 16f044b..a648667 100755 --- a/bin/default_pythons +++ b/bin/default_pythons @@ -1,11 +1,21 @@ #!/usr/bin/env bash -DEFAULT_PYTHON_VERSION="python-3.6.9" -LATEST_38="python-3.8.0" -LATEST_37="python-3.7.5" -LATEST_36="python-3.6.9" +DEFAULT_PYTHON_VERSION="python-3.6.10" +LATEST_38="python-3.8.1" +LATEST_37="python-3.7.6" +LATEST_36="python-3.6.10" LATEST_35="python-3.5.7" LATEST_34="python-3.4.10" LATEST_27="python-2.7.17" +PYPY_36="pypy3.6-7.2.0" +PYPY_27="pypy2.7-7.2.0" -export DEFAULT_PYTHON_VERSION LATEST_38 LATEST_37 LATEST_36 LATEST_35 LATEST_34 LATEST_27 +export DEFAULT_PYTHON_VERSION \ + LATEST_38 \ + LATEST_37 \ + LATEST_36 \ + LATEST_35 \ + LATEST_34 \ + LATEST_27 \ + PYPY_36 \ + PYPY_27 diff --git a/bin/steps/python b/bin/steps/python index 2625b44..8790f61 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -9,6 +9,10 @@ VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz" SECURITY_UPDATE="Python has released a security update! Please consider upgrading to" +ONLY_SUPPORTED_2_VERSION="Only the latest version of Python 2 is supported on the platform. Please consider upgrading to" + +PYTHON_2_EOL_UPDATE="Python 2 has reached it's community EOL. Upgrade your Python runtime to maintain a secure application as soon as possible." + # check if runtime exists if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then if [[ "$PYTHON_VERSION" == $PY38* ]]; then @@ -48,8 +52,26 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then fi if [[ "$PYTHON_VERSION" == $PY27* ]]; then # security update note + if [[ "$(date "+%Y")" -gt "2019" ]]; then + puts-warn "$PYTHON_2_EOL_UPDATE" + echo " Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq" + fi if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then - puts-warn "$SECURITY_UPDATE" "$LATEST_27" + puts-warn "$ONLY_SUPPORTED_2_VERSION" "$LATEST_27" + echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" + fi + fi + if [[ "$PYTHON_VERSION" == $PYPY27* ]]; then + # security update note + if [ "$PYTHON_VERSION" != "$PYPY_27" ]; then + puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_27}?" + echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" + fi + fi + if [[ "$PYTHON_VERSION" == $PYPY36* ]]; then + # security update note + if [ "$PYTHON_VERSION" != "$PYPY_36" ]; then + puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_36}?" echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" fi fi diff --git a/bin/steps/sqlite3 b/bin/steps/sqlite3 index 3faae6e..2d8dc91 100755 --- a/bin/steps/sqlite3 +++ b/bin/steps/sqlite3 @@ -3,21 +3,8 @@ # shellcheck source=bin/utils source "$BIN_DIR/utils" -sqlite3_version() { - if [ "$STACK" = "cedar-14" ]; then - SQLITE3_VERSION="3.8.2-1ubuntu2.2" - elif [ "$STACK" = "heroku-16" ]; then - SQLITE3_VERSION="3.11.0-1ubuntu1.2" - else - SQLITE3_VERSION=${SQLITE3_VERSION:-$(dpkg -s libsqlite3-0 | grep Version | sed 's/Version: //')} - fi - - export SQLITE3_VERSION -} - sqlite3_install() { HEROKU_PYTHON_DIR="$1" - SQLITE3_VERSION="$2" HEADERS_ONLY="$3" mkdir -p "$HEROKU_PYTHON_DIR" @@ -35,9 +22,9 @@ sqlite3_install() { apt-get $APT_OPTIONS update > /dev/null 2>&1 if [ -z "$HEADERS_ONLY" ]; then - apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION" sqlite3="$SQLITE3_VERSION" > /dev/null 2>&1 + apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev sqlite3 > /dev/null 2>&1 else - apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION" + apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev fi find "$APT_CACHE_DIR/archives/" -name "*.deb" -exec dpkg -x {} "$HEROKU_PYTHON_DIR/sqlite3/" \; @@ -70,7 +57,6 @@ sqlite3_install() { } buildpack_sqlite3_install() { - sqlite3_version HEROKU_PYTHON_DIR="$BUILD_DIR/.heroku/python" SQLITE3_VERSION_FILE="$BUILD_DIR/.heroku/python-sqlite3-version" @@ -78,16 +64,15 @@ buildpack_sqlite3_install() { INSTALLED_SQLITE3_VERSION=$(cat "$SQLITE3_VERSION_FILE") fi - # python version check - if python_sqlite3_check "$PYTHON_VERSION"; then - # only install if the sqlite3 version has changed - if [ "$INSTALLED_SQLITE3_VERSION" != "$SQLITE3_VERSION" ]; then - puts-step "Installing SQLite3" - sqlite3_install "$BUILD_DIR/.heroku/python" "$SQLITE3_VERSION" + puts-step "Installing SQLite3" - # save version installed - mkdir -p "$CACHE_DIR/.heroku/" - echo "$SQLITE3_VERSION" > "$CACHE_DIR/.heroku/python-sqlite3-version" - fi + if sqlite3_install "$BUILD_DIR/.heroku/python" ; then + echo "Sqlite3 successfully installed." + mcount "success.python.sqlite3" + else + echo "Sqlite3 failed to install." + mcount "failure.python.sqlite3" fi + + mkdir -p "$CACHE_DIR/.heroku/" } diff --git a/builds/runtimes/pypy2.7 b/builds/runtimes/pypy2.7 new file mode 100755 index 0000000..698aa17 --- /dev/null +++ b/builds/runtimes/pypy2.7 @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# fail hard +set -o pipefail +# fail harder +set -eu + +OUT_PREFIX=$1 + +dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 +dep_name=$(basename $BASH_SOURCE) # this is us +dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name +dep_package=${dep_name}-v${dep_version} # it's always "pypy2-…" +dep_dirname=${dep_package}-linux64 +dep_archive_name=${dep_dirname}.tar.bz2 +dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name} + +echo "Building PyPy…" +echo "${dep_url}" + +curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name + +ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python" diff --git a/builds/runtimes/pypy2.7-7.2.0 b/builds/runtimes/pypy2.7-7.2.0 new file mode 100755 index 0000000..40761e9 --- /dev/null +++ b/builds/runtimes/pypy2.7-7.2.0 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/pypy2.7 diff --git a/builds/runtimes/pypy3.6 b/builds/runtimes/pypy3.6 new file mode 100755 index 0000000..f24f806 --- /dev/null +++ b/builds/runtimes/pypy3.6 @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# fail hard +set -o pipefail +# fail harder +set -eu + +OUT_PREFIX=$1 + +dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 +dep_name=$(basename $BASH_SOURCE) # this is us +dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name +dep_package=${dep_name}${dep_version_prefix:-}-v${dep_version}${dep_version_suffix:-} +dep_dirname=${dep_package}-linux64 +dep_archive_name=${dep_dirname}.tar.bz2 +dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name} + +echo "Building PyPy3…" +echo "${dep_url}" + +curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name + +ln "$OUT_PREFIX/bin/pypy3" "$OUT_PREFIX/bin/python" diff --git a/builds/runtimes/pypy3.6-7.2.0 b/builds/runtimes/pypy3.6-7.2.0 new file mode 100755 index 0000000..e429be8 --- /dev/null +++ b/builds/runtimes/pypy3.6-7.2.0 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/pypy3.6 diff --git a/builds/runtimes/python-3.6.10 b/builds/runtimes/python-3.6.10 new file mode 100755 index 0000000..ad41723 --- /dev/null +++ b/builds/runtimes/python-3.6.10 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python-3.7.6 b/builds/runtimes/python-3.7.6 new file mode 100755 index 0000000..ad41723 --- /dev/null +++ b/builds/runtimes/python-3.7.6 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python-3.8.1 b/builds/runtimes/python-3.8.1 new file mode 100755 index 0000000..ad41723 --- /dev/null +++ b/builds/runtimes/python-3.8.1 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python3 b/builds/runtimes/python3 index 1b6d152..6e7c517 100755 --- a/builds/runtimes/python3 +++ b/builds/runtimes/python3 @@ -16,6 +16,9 @@ python_version=${BASE^} # this gives us only the filename with version number version_number=$(echo "$python_version" | cut -d- -f2) # this returns just X.X.X dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz +echo "Building Python 3..." +echo "Pulling from source: ${dep_url}" + curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}" mv "${OUT_PREFIX}/${python_version}" src cd src @@ -31,9 +34,4 @@ find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm - # Remove spare / LOCATION=${OUT_PREFIX%?} -# Create links to SQLITE headers so Python can call them at runtime -mkdir -p ${OUT_PREFIX}/include ${OUT_PREFIX}/lib/x86_64-linux-gnu -cp /usr/include/sqlite3*.h ${OUT_PREFIX}/include -ln -fs $(realpath /usr/lib/x86_64-linux-gnu/libsqlite3.so) ${OUT_PREFIX}/lib/x86_64-linux-gnu/libsqlite3.so - ln $LOCATION/bin/python3 $LOCATION/bin/python diff --git a/test/fixtures/pypy2_7/requirements.txt b/test/fixtures/pypy2_7/requirements.txt new file mode 100644 index 0000000..d3e4ba5 --- /dev/null +++ b/test/fixtures/pypy2_7/requirements.txt @@ -0,0 +1 @@ +django diff --git a/test/fixtures/pypy2_7/runtime.txt b/test/fixtures/pypy2_7/runtime.txt new file mode 100644 index 0000000..c502b68 --- /dev/null +++ b/test/fixtures/pypy2_7/runtime.txt @@ -0,0 +1 @@ +pypy2.7-7.2.0 diff --git a/test/fixtures/pypy3_6/requirements.txt b/test/fixtures/pypy3_6/requirements.txt new file mode 100644 index 0000000..d3e4ba5 --- /dev/null +++ b/test/fixtures/pypy3_6/requirements.txt @@ -0,0 +1 @@ +django diff --git a/test/fixtures/pypy3_6/runtime.txt b/test/fixtures/pypy3_6/runtime.txt new file mode 100644 index 0000000..e1af38f --- /dev/null +++ b/test/fixtures/pypy3_6/runtime.txt @@ -0,0 +1 @@ +pypy3.6-7.2.0 diff --git a/test/run-deps b/test/run-deps index bd03fa7..29d7f8f 100755 --- a/test/run-deps +++ b/test/run-deps @@ -46,6 +46,13 @@ testPysqlite() { assertCapturedSuccess } +testSqliteInstall() { + compile "pythonDefault" + assertCaptured "Sqlite3 successfully installed." + assertNotCaptured "Sqlite3 failed to install." + assertCapturedSuccess +} + testCffi() { compile "cffi" assertCaptured "cffi" diff --git a/test/run-features b/test/run-features index 66fac4e..c3ee1f2 100755 --- a/test/run-features +++ b/test/run-features @@ -16,7 +16,6 @@ testSmartRequirements() { assertFile "requests" ".heroku/python/requirements-declared.txt" assertCapturedSuccess compile "psycopg2" "$cache_dir" - assertCaptured "Uninstalling requests" assertFile "psycopg2" ".heroku/python/requirements-declared.txt" assertCapturedSuccess } diff --git a/test/run-versions b/test/run-versions index 0698b7c..c207be1 100755 --- a/test/run-versions +++ b/test/run-versions @@ -18,6 +18,11 @@ testPython2() { echo $LATEST_27 > "runtime.txt" compile "python2" assertCaptured $LATEST_27 + if [[ $(date "+%Y") > "2019" ]]; then + assertCaptured "python-2-7-eol-faq"; + else + assertNotCaptured "python-2-7-eol-faq"; + fi assertNotCaptured "security update" assertCaptured "Installing SQLite3" assertCapturedSuccess @@ -26,7 +31,12 @@ testPython2() { testPython2_warn() { compile "python2_warn" assertCaptured "python-2.7.15" - assertCaptured "security update!" + if [[ $(date "+%Y") > "2019" ]]; then + assertCaptured "python-2-7-eol-faq"; + else + assertNotCaptured "python-2-7-eol-faq"; + fi + assertCaptured "Only the latest version" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -166,6 +176,28 @@ testPython3_8_fail() { assertCapturedError } +testPypy3_6() { + compile "pypy3_6" + if [[ $STACK = "cedar-14" ]]; then + assertCapturedError + else + assertCaptured "Installing pypy" + assertCaptured "$PYPY_36" + assertCapturedSuccess + fi +} + +testPypy2_7() { + compile "pypy2_7" + if [[ $STACK = "cedar-14" ]]; then + assertCapturedError + else + assertCaptured "Installing pypy" + assertCaptured "$PYPY_27" + assertCapturedSuccess + fi +} + pushd $(dirname 0) >/dev/null popd >/dev/null