diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b74422..043667f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,22 @@ # Master +- Python 3.8.2 now available. + + -------------------------------------------------------------------------------- +# 163 (2020-02-20) + +- Update requirements.txt builds to use Pip 20.0.2 +- Download get-pip.py to tmpdir instead of root dir + +# 163 (2019-12-23) + +- New pythons released: + Python 3.8.1, 3.7.6, 3.6.10 (CPython) + Beta Release: Pypy 2.7 and 3.6, version 7.2.0 + # 162 (2019-12-06) - Bug fix: fragile sqlite3 install diff --git a/bin/compile b/bin/compile index 9319463..4a214c2 100755 --- a/bin/compile +++ b/bin/compile @@ -58,11 +58,31 @@ PY36="python-3.6" PY35="python-3.5" PY34="python-3.4" PY27="python-2.7" +PYPY27="pypy2.7" +PYPY36="pypy3.6" # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)? DEFAULT_PYTHON_STACK="cedar-14" # If pip doesn't match this version (the version we install), run the installer. -PIP_UPDATE="9.0.2" +PIP_UPDATE="20.0.2" + +for file in "$BUILD_DIR/runtime.txt" "$CACHE_DIR/.heroku/python-version" ; do + [ -f "$file" ] || continue + + version=$(tr -d '[:space:]' < "$file") + + case "$version" in "$PY34"*) + # Python 3.4 support was dropped in pip >= 19.2. + PIP_UPDATE="19.1.1" + break + ;; + esac +done + +if [[ -f "$BUILD_DIR/Pipfile" ]]; then + # Do not force pipenv users to re-install pipenv locally. + PIP_UPDATE="9.0.2" +fi export DEFAULT_PYTHON_STACK PIP_UPDATE export PY37 PY36 PY35 PY27 PY34 diff --git a/bin/default_pythons b/bin/default_pythons index 16f044b..6c6d4a1 100755 --- a/bin/default_pythons +++ b/bin/default_pythons @@ -1,11 +1,21 @@ #!/usr/bin/env bash -DEFAULT_PYTHON_VERSION="python-3.6.9" -LATEST_38="python-3.8.0" -LATEST_37="python-3.7.5" -LATEST_36="python-3.6.9" +DEFAULT_PYTHON_VERSION="python-3.6.10" +LATEST_38="python-3.8.2" +LATEST_37="python-3.7.6" +LATEST_36="python-3.6.10" LATEST_35="python-3.5.7" LATEST_34="python-3.4.10" LATEST_27="python-2.7.17" +PYPY_36="pypy3.6-7.2.0" +PYPY_27="pypy2.7-7.2.0" -export DEFAULT_PYTHON_VERSION LATEST_38 LATEST_37 LATEST_36 LATEST_35 LATEST_34 LATEST_27 +export DEFAULT_PYTHON_VERSION \ + LATEST_38 \ + LATEST_37 \ + LATEST_36 \ + LATEST_35 \ + LATEST_34 \ + LATEST_27 \ + PYPY_36 \ + PYPY_27 diff --git a/bin/steps/python b/bin/steps/python index 2625b44..c06db37 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -9,6 +9,10 @@ VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz" SECURITY_UPDATE="Python has released a security update! Please consider upgrading to" +ONLY_SUPPORTED_2_VERSION="Only the latest version of Python 2 is supported on the platform. Please consider upgrading to" + +PYTHON_2_EOL_UPDATE="Python 2 has reached it's community EOL. Upgrade your Python runtime to maintain a secure application as soon as possible." + # check if runtime exists if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then if [[ "$PYTHON_VERSION" == $PY38* ]]; then @@ -48,8 +52,26 @@ if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then fi if [[ "$PYTHON_VERSION" == $PY27* ]]; then # security update note + if [[ "$(date "+%Y")" -gt "2019" ]]; then + puts-warn "$PYTHON_2_EOL_UPDATE" + echo " Learn More: https://devcenter.heroku.com/articles/python-2-7-eol-faq" + fi if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then - puts-warn "$SECURITY_UPDATE" "$LATEST_27" + puts-warn "$ONLY_SUPPORTED_2_VERSION" "$LATEST_27" + echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" + fi + fi + if [[ "$PYTHON_VERSION" == $PYPY27* ]]; then + # security update note + if [ "$PYTHON_VERSION" != "$PYPY_27" ]; then + puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_27}?" + echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" + fi + fi + if [[ "$PYTHON_VERSION" == $PYPY36* ]]; then + # security update note + if [ "$PYTHON_VERSION" != "$PYPY_36" ]; then + puts-warn "Could not find that Pypy version. Did you mean" "${PYPY_36}?" echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" fi fi @@ -106,9 +128,10 @@ fi # Heroku uses the get-pip utility maintained by the Python community to vendor Pip. # https://github.com/pypa/get-pip -GETPIP="https://lang-python.s3.amazonaws.com/etc/get-pip.py" +GETPIP_URL="https://lang-python.s3.amazonaws.com/etc/get-pip.py" +GETPIP_PY="${TMPDIR:-/tmp}/get-pip.py" -if ! curl -s "${GETPIP}" -o "$ROOT_DIR/get-pip.py" &> /dev/null; then +if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then mcount "failure.python.get-pip" echo "Failed to pull down get-pip" exit 1 @@ -123,7 +146,7 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then rm -fr /app/.heroku/python/lib/python*/site-packages/pip-* rm -fr /app/.heroku/python/lib/python*/site-packages/setuptools-* - /app/.heroku/python/bin/python "$ROOT_DIR/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null + /app/.heroku/python/bin/python "$GETPIP_PY" pip=="$PIP_UPDATE" &> /dev/null /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null fi diff --git a/builds/runtimes/pypy2.7 b/builds/runtimes/pypy2.7 new file mode 100755 index 0000000..698aa17 --- /dev/null +++ b/builds/runtimes/pypy2.7 @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# fail hard +set -o pipefail +# fail harder +set -eu + +OUT_PREFIX=$1 + +dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 +dep_name=$(basename $BASH_SOURCE) # this is us +dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name +dep_package=${dep_name}-v${dep_version} # it's always "pypy2-…" +dep_dirname=${dep_package}-linux64 +dep_archive_name=${dep_dirname}.tar.bz2 +dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name} + +echo "Building PyPy…" +echo "${dep_url}" + +curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name + +ln "$OUT_PREFIX/bin/pypy" "$OUT_PREFIX/bin/python" diff --git a/builds/runtimes/pypy2.7-7.2.0 b/builds/runtimes/pypy2.7-7.2.0 new file mode 100755 index 0000000..40761e9 --- /dev/null +++ b/builds/runtimes/pypy2.7-7.2.0 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/pypy2.7 diff --git a/builds/runtimes/pypy3.6 b/builds/runtimes/pypy3.6 new file mode 100755 index 0000000..f24f806 --- /dev/null +++ b/builds/runtimes/pypy3.6 @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# fail hard +set -o pipefail +# fail harder +set -eu + +OUT_PREFIX=$1 + +dep_formula=${0#$WORKSPACE_DIR/} # this is the original script, e.g. pypy-5.3.1 +dep_name=$(basename $BASH_SOURCE) # this is us +dep_version=${dep_formula##*"/${dep_name}-"} # "subtract" our name from full version name +dep_package=${dep_name}${dep_version_prefix:-}-v${dep_version}${dep_version_suffix:-} +dep_dirname=${dep_package}-linux64 +dep_archive_name=${dep_dirname}.tar.bz2 +dep_url=https://bitbucket.org/pypy/pypy/downloads/${dep_archive_name} + +echo "Building PyPy3…" +echo "${dep_url}" + +curl -L "${dep_url}" | tar jx -C "${OUT_PREFIX}" --strip-components 1 # extract to $OUT_PREFIX, drop the first directory level, which is the archive name + +ln "$OUT_PREFIX/bin/pypy3" "$OUT_PREFIX/bin/python" diff --git a/builds/runtimes/pypy3.6-7.2.0 b/builds/runtimes/pypy3.6-7.2.0 new file mode 100755 index 0000000..e429be8 --- /dev/null +++ b/builds/runtimes/pypy3.6-7.2.0 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/pypy3.6 diff --git a/builds/runtimes/python-3.6.10 b/builds/runtimes/python-3.6.10 new file mode 100755 index 0000000..ad41723 --- /dev/null +++ b/builds/runtimes/python-3.6.10 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python-3.7.6 b/builds/runtimes/python-3.7.6 new file mode 100755 index 0000000..ad41723 --- /dev/null +++ b/builds/runtimes/python-3.7.6 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python-3.8.1 b/builds/runtimes/python-3.8.1 new file mode 100755 index 0000000..ad41723 --- /dev/null +++ b/builds/runtimes/python-3.8.1 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python-3.8.2 b/builds/runtimes/python-3.8.2 new file mode 100755 index 0000000..ad41723 --- /dev/null +++ b/builds/runtimes/python-3.8.2 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python-3.8.3 b/builds/runtimes/python-3.8.3 new file mode 100755 index 0000000..ad41723 --- /dev/null +++ b/builds/runtimes/python-3.8.3 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +source $(dirname $0)/python3 diff --git a/builds/runtimes/python3 b/builds/runtimes/python3 index 1b6d152..6e7c517 100755 --- a/builds/runtimes/python3 +++ b/builds/runtimes/python3 @@ -16,6 +16,9 @@ python_version=${BASE^} # this gives us only the filename with version number version_number=$(echo "$python_version" | cut -d- -f2) # this returns just X.X.X dep_url=https://python.org/ftp/python/${version_number}/${python_version}.tgz +echo "Building Python 3..." +echo "Pulling from source: ${dep_url}" + curl -L "${dep_url}" | tar xz -C "${OUT_PREFIX}" mv "${OUT_PREFIX}/${python_version}" src cd src @@ -31,9 +34,4 @@ find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm - # Remove spare / LOCATION=${OUT_PREFIX%?} -# Create links to SQLITE headers so Python can call them at runtime -mkdir -p ${OUT_PREFIX}/include ${OUT_PREFIX}/lib/x86_64-linux-gnu -cp /usr/include/sqlite3*.h ${OUT_PREFIX}/include -ln -fs $(realpath /usr/lib/x86_64-linux-gnu/libsqlite3.so) ${OUT_PREFIX}/lib/x86_64-linux-gnu/libsqlite3.so - ln $LOCATION/bin/python3 $LOCATION/bin/python diff --git a/requirements.txt b/requirements.txt index ad7e217..ae6edbf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ docopt==0.6.2 -bob-builder +bob-builder==0.0.17 boto==2.48.0 diff --git a/spec/hatchet/python_spec.rb b/spec/hatchet/python_spec.rb index 9651d60..98f0018 100644 --- a/spec/hatchet/python_spec.rb +++ b/spec/hatchet/python_spec.rb @@ -4,7 +4,7 @@ describe "Python!!!!!!!!!!!" do it "🐍" do Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app| expect(app.output).to match(/Installing pip/) - expect(app.run('python -V')).to match(/3.6.9/) + expect(app.run('python -V')).to match(/3.6.10/) end end end diff --git a/test/fixtures/pypy2_7/requirements.txt b/test/fixtures/pypy2_7/requirements.txt new file mode 100644 index 0000000..d3e4ba5 --- /dev/null +++ b/test/fixtures/pypy2_7/requirements.txt @@ -0,0 +1 @@ +django diff --git a/test/fixtures/pypy2_7/runtime.txt b/test/fixtures/pypy2_7/runtime.txt new file mode 100644 index 0000000..c502b68 --- /dev/null +++ b/test/fixtures/pypy2_7/runtime.txt @@ -0,0 +1 @@ +pypy2.7-7.2.0 diff --git a/test/fixtures/pypy3_6/requirements.txt b/test/fixtures/pypy3_6/requirements.txt new file mode 100644 index 0000000..d3e4ba5 --- /dev/null +++ b/test/fixtures/pypy3_6/requirements.txt @@ -0,0 +1 @@ +django diff --git a/test/fixtures/pypy3_6/runtime.txt b/test/fixtures/pypy3_6/runtime.txt new file mode 100644 index 0000000..e1af38f --- /dev/null +++ b/test/fixtures/pypy3_6/runtime.txt @@ -0,0 +1 @@ +pypy3.6-7.2.0 diff --git a/test/fixtures/python3_8/runtime.txt b/test/fixtures/python3_8/runtime.txt index 73b1cf8..724c203 100644 --- a/test/fixtures/python3_8/runtime.txt +++ b/test/fixtures/python3_8/runtime.txt @@ -1 +1 @@ -python-3.8.0 +python-3.8.2 diff --git a/test/fixtures/python3_8_warn/requirements.txt b/test/fixtures/python3_8_warn/requirements.txt new file mode 100644 index 0000000..f229360 --- /dev/null +++ b/test/fixtures/python3_8_warn/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/test/fixtures/python3_8_warn/runtime.txt b/test/fixtures/python3_8_warn/runtime.txt new file mode 100644 index 0000000..73b1cf8 --- /dev/null +++ b/test/fixtures/python3_8_warn/runtime.txt @@ -0,0 +1 @@ +python-3.8.0 diff --git a/test/run-versions b/test/run-versions index 0698b7c..75f165b 100755 --- a/test/run-versions +++ b/test/run-versions @@ -18,6 +18,11 @@ testPython2() { echo $LATEST_27 > "runtime.txt" compile "python2" assertCaptured $LATEST_27 + if [[ $(date "+%Y") > "2019" ]]; then + assertCaptured "python-2-7-eol-faq"; + else + assertNotCaptured "python-2-7-eol-faq"; + fi assertNotCaptured "security update" assertCaptured "Installing SQLite3" assertCapturedSuccess @@ -26,7 +31,12 @@ testPython2() { testPython2_warn() { compile "python2_warn" assertCaptured "python-2.7.15" - assertCaptured "security update!" + if [[ $(date "+%Y") > "2019" ]]; then + assertCaptured "python-2-7-eol-faq"; + else + assertNotCaptured "python-2-7-eol-faq"; + fi + assertCaptured "Only the latest version" assertCaptured "Installing SQLite3" assertCapturedSuccess } @@ -147,6 +157,18 @@ testPython3_7_fail() { assertCapturedError } +testPython3_7_warn() { + compile "python3_8_warn" + if [[ $STACK = "cedar-14" ]]; then + assertCapturedError + else + assertCaptured "python-3.8.0" + assertCaptured "security update!" + assertCaptured "Installing SQLite3" + assertCapturedSuccess + fi +} + testPython3_8() { updateVersion "python3_8" $LATEST_38 compile "python3_8" @@ -166,6 +188,28 @@ testPython3_8_fail() { assertCapturedError } +testPypy3_6() { + compile "pypy3_6" + if [[ $STACK = "cedar-14" ]]; then + assertCapturedError + else + assertCaptured "Installing pypy" + assertCaptured "$PYPY_36" + assertCapturedSuccess + fi +} + +testPypy2_7() { + compile "pypy2_7" + if [[ $STACK = "cedar-14" ]]; then + assertCapturedError + else + assertCaptured "Installing pypy" + assertCaptured "$PYPY_27" + assertCapturedSuccess + fi +} + pushd $(dirname 0) >/dev/null popd >/dev/null diff --git a/vendor/pip-pop/pip-diff b/vendor/pip-pop/pip-diff index 2bb1877..8fc7d6f 100755 --- a/vendor/pip-pop/pip-diff +++ b/vendor/pip-pop/pip-diff @@ -12,9 +12,24 @@ Options: """ import os from docopt import docopt -from pip.req import parse_requirements -from pip.index import PackageFinder -from pip._vendor.requests import session + +try: # pip >= 10 + from pip._internal.req import parse_requirements + from pip._internal.download import PipSession as session + + def PackageFinder(find_links, index_urls, session=None): + from pip._internal.index import PackageFinder + from pip._internal.models.search_scope import SearchScope + from pip._internal.models.selection_prefs import SelectionPreferences + + search_scope = SearchScope.create(find_links, index_urls) + selection_prefs = SelectionPreferences(allow_yanked=False) + return PackageFinder.create(search_scope, selection_prefs, session=session) + +except ImportError: # pip <= 9.0.3 + from pip.req import parse_requirements + from pip.index import PackageFinder + from pip._vendor.requests import session requests = session() diff --git a/vendor/pip-pop/pip-grep b/vendor/pip-pop/pip-grep index d55000a..8e3cf61 100755 --- a/vendor/pip-pop/pip-grep +++ b/vendor/pip-pop/pip-grep @@ -10,9 +10,25 @@ Options: import os import sys from docopt import docopt -from pip.req import parse_requirements -from pip.index import PackageFinder -from pip._vendor.requests import session + +try: # pip >= 10 + from pip._internal.req import parse_requirements + from pip._internal.download import PipSession as session + + def PackageFinder(find_links, index_urls, session=None): + from pip._internal.index import PackageFinder + from pip._internal.models.search_scope import SearchScope + from pip._internal.models.selection_prefs import SelectionPreferences + + search_scope = SearchScope.create(find_links, index_urls) + selection_prefs = SelectionPreferences(allow_yanked=False) + return PackageFinder.create(search_scope, selection_prefs, session=session) + +except ImportError: # pip <= 9.0.3 + from pip.req import parse_requirements + from pip.index import PackageFinder + from pip._vendor.requests import session + requests = session()