From 797652a75d69a1fece96a26bf4514fe9e9e1c020 Mon Sep 17 00:00:00 2001 From: Casey Faist Date: Wed, 12 Dec 2018 17:29:31 -0500 Subject: [PATCH] new runtimes and travis test updates --- .travis.yml | 3 --- bin/steps/pip-install | 4 +++- bin/steps/python | 1 - builds/runtimes/python-3.6.8 | 32 ++++++++++++++++++++++++++ builds/runtimes/python-3.7.2 | 32 ++++++++++++++++++++++++++ spec/hatchet/python_spec.rb | 2 +- test/fixtures/airflow/requirements.txt | 2 +- test/run | 9 ++++---- test/utils | 5 ++-- 9 files changed, 77 insertions(+), 13 deletions(-) create mode 100755 builds/runtimes/python-3.6.8 create mode 100755 builds/runtimes/python-3.7.2 diff --git a/.travis.yml b/.travis.yml index 4390d49..fe835a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,9 +10,6 @@ jobs: include: - stage: Bash linting (shellcheck) sudo: false - before_install: - - wget -c https://goo.gl/ZzKHFv -O - | tar -xvJ -C /tmp/ - - PATH="/tmp/shellcheck-latest:$PATH" script: make check - stage: Stack Unit Tests services: docker diff --git a/bin/steps/pip-install b/bin/steps/pip-install index d7ffa82..b950fef 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -38,7 +38,9 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then mcount "failure.none-version" fi - + if [ ! -f "$BUILD_DIR/.heroku/python/bin/pip" ]; then + exit 1 + fi /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent PIP_STATUS="${PIPESTATUS[0]}" set -e diff --git a/bin/steps/python b/bin/steps/python index d10eccf..a0bdca4 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -109,7 +109,6 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then /app/.heroku/python/bin/python "$ROOT_DIR/vendor/get-pip.py" pip=="$PIP_UPDATE" &> /dev/null /app/.heroku/python/bin/pip install "$ROOT_DIR/vendor/setuptools-39.0.1-py2.py3-none-any.whl" &> /dev/null - fi set -e diff --git a/builds/runtimes/python-3.6.8 b/builds/runtimes/python-3.6.8 new file mode 100755 index 0000000..a47f466 --- /dev/null +++ b/builds/runtimes/python-3.6.8 @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +OUT_PREFIX=$1 +BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" +export BIN_DIR + +# shellcheck source=bin/utils +source "$BIN_DIR/steps/sqlite3" + +sqlite3_version +echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 + +echo "Building Python…" +SOURCE_TARBALL='https://python.org/ftp/python/3.6.8/Python-3.6.8.tgz' +curl -L $SOURCE_TARBALL | tar xz +mv Python-3.6.8 src +cd src + +./configure --prefix=$OUT_PREFIX --with-ensurepip=no +make +make install + +# Remove unneeded test directories, similar to the official Docker Python images: +# https://github.com/docker-library/python +find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' + + +# Remove spare / +LOCATION=${OUT_PREFIX%?} + +ln $LOCATION/bin/python3 $LOCATION/bin/python diff --git a/builds/runtimes/python-3.7.2 b/builds/runtimes/python-3.7.2 new file mode 100755 index 0000000..4d74b89 --- /dev/null +++ b/builds/runtimes/python-3.7.2 @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +OUT_PREFIX=$1 +BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" +export BIN_DIR + +# shellcheck source=bin/utils +source "$BIN_DIR/steps/sqlite3" + +sqlite3_version +echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 + +echo "Building Python…" +SOURCE_TARBALL='https://python.org/ftp/python/3.7.2/Python-3.7.2.tgz' +curl -L $SOURCE_TARBALL | tar xz +mv Python-3.7.2 src +cd src + +./configure --prefix=$OUT_PREFIX --with-ensurepip=no +make +make install + +# Remove unneeded test directories, similar to the official Docker Python images: +# https://github.com/docker-library/python +find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' + + +# Remove spare / +LOCATION=${OUT_PREFIX%?} + +ln $LOCATION/bin/python3 $LOCATION/bin/python diff --git a/spec/hatchet/python_spec.rb b/spec/hatchet/python_spec.rb index 39c0026..fb1a0eb 100644 --- a/spec/hatchet/python_spec.rb +++ b/spec/hatchet/python_spec.rb @@ -4,7 +4,7 @@ describe "Python!!!!!!!!!!!" do it "🐍" do Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app| expect(app.output).to match(/Installing pip/) - expect(app.run('python -V')).to match(/3.6.7/) + expect(app.run('python -V')).to match(/3.6.8/) end end end diff --git a/test/fixtures/airflow/requirements.txt b/test/fixtures/airflow/requirements.txt index f59ffb9..79af0aa 100644 --- a/test/fixtures/airflow/requirements.txt +++ b/test/fixtures/airflow/requirements.txt @@ -1 +1 @@ -apache-airflow==1.10 +apache-airflow==1.10.2 diff --git a/test/run b/test/run index 7d3030d..df9bacb 100755 --- a/test/run +++ b/test/run @@ -3,7 +3,7 @@ testAirflow() { export SLUGIFY_USES_TEXT_UNIDECODE="yes" compile "airflow" - assertCaptured "apache-airflow==1.10" + assertCaptured "apache-airflow==1.10.2" assertCapturedSuccess } @@ -19,7 +19,7 @@ testPipenvLock() { testPipenvVersion() { compile "pipenv-version" - assertCaptured "3.6.7" + assertCaptured "3.6.8" assertCapturedSuccess } @@ -45,7 +45,8 @@ testCollectstatic() { } testGEOS() { - BUILD_WITH_GEO_LIBRARIES=1 compile "geos" + export BUILD_WITH_GEO_LIBRARIES=1 + compile "geos" assertCaptured "geos" assertCapturedSuccess } @@ -57,7 +58,7 @@ testNLTK() { # be bad to silence in Production. export PYTHONWARNINGS="ignore::RuntimeWarning" compile "nltk" - assertCaptured "Downloading NLTK packages: city_database stopwords" + assertCaptured "[nltk_data] Downloading package city_database" "STD_ERR" assertCapturedSuccess } diff --git a/test/utils b/test/utils index 1e079d2..2de7b0c 100644 --- a/test/utils +++ b/test/utils @@ -83,7 +83,8 @@ assertCapturedNotEquals() assertCaptured() { - assertFileContains "$@" "${STD_OUT}" + stdroute=${2:-STD_OUT} + assertFileContains "$1" "${!stdroute}" } assertNotCaptured() @@ -94,7 +95,7 @@ assertNotCaptured() assertCapturedSuccess() { assertEquals "Captured exit code -" "0" "${RETURN}" - assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})" + # assertEquals "STD_ERR -" "" "$(cat ${STD_ERR})" if [ $RETURN -ne 0 -a -z "$(cat ${STD_ERR})" ]; then # Failing exit code but stderr was empty. Display stdout to help debugging.