diff --git a/.gitignore b/.gitignore index 1ffa3cb..b1e1f23 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ buildpack/* builds/dockerenv.staging* builds/dockerenv.production + +test/scratch diff --git a/.travis.yml b/.travis.yml index 102e16e..d5449a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,11 +10,11 @@ before_script: script: - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile . - - docker run --rm -e "STACK=cedar-14" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-cedar-14 bash $TESTFOLDER + - docker run --rm -e "STACK=cedar-14" travis-build-cedar-14 bash $TESTFOLDER - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile . - - docker run --rm -e "STACK=heroku-16" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-16 bash $TESTFOLDER + - docker run --rm -e "STACK=heroku-16" travis-build-heroku-16 bash $TESTFOLDER - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile . - - docker run --rm -e "STACK=heroku-18" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-18 bash $TESTFOLDER + - docker run --rm -e "STACK=heroku-18" travis-build-heroku-18 bash $TESTFOLDER jobs: include: diff --git a/CHANGELOG.md b/CHANGELOG.md index 64263b4..e5146f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ # Master +- Add failcase for cache busting +- Bugfix: Clearing pip dependencies - Correct ftp to https in vendored file - Warn for Django 1.11 approaching EOL, provide link to roadmap [fixed detection] diff --git a/bin/compile b/bin/compile index 4a214c2..170026c 100755 --- a/bin/compile +++ b/bin/compile @@ -259,17 +259,8 @@ mtime "python.install.time" "${start}" # shellcheck source=bin/steps/pipenv source "$BIN_DIR/steps/pipenv" -# Uninstall removed dependencies with Pip. -# The buildpack will automatically remove any declared dependencies (in requirements.txt) -# that were explicitly removed. This machinery is a bit complex, but it is not complicated. -(( start=$(nowms) )) -# shellcheck source=bin/steps/pip-uninstall -source "$BIN_DIR/steps/pip-uninstall" -mtime "pip.uninstall.time" "${start}" - # If no requirements.txt file given, assume `setup.py develop` is intended. # This allows for people to ship a setup.py application to Heroku -# (which is rare, but I vouch that it should work!) if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then echo "-e ." > requirements.txt diff --git a/bin/steps/README.MD b/bin/steps/README.MD new file mode 100644 index 0000000..3c23cf7 --- /dev/null +++ b/bin/steps/README.MD @@ -0,0 +1,28 @@ +# Python Buildpack Install Steps + +TODO: Add context on Python install steps, such as why symlinking vs copying + +## Installing the Pip tool + +The Python Buildpack uses a tool called `get-pip` to install the pip tool. This +is done in the `python` script. + +This is in part because Python historically did not come with pip by default. + +## Installing Python packages using Pip + +### Convention: Use `python` process to invoke Pip + +We don't use this convention (yet) but this is an upcoming change being considered. + +This is a bigger concern on Windows than it is in Linux environments, but an +emerging convention in the Python community is to invoke pip using: + +``` +python3 -m pip [options] +``` + +Invoking pip this way ensures correct location - python knows where these +packages are stored because it put them there (defaults to Python's pathing info). + +All normal command line options are available using this method. diff --git a/bin/steps/pip-install b/bin/steps/pip-install index 481ffc6..b33422a 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -47,6 +47,7 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then if [ ! -f "$BUILD_DIR/.heroku/python/bin/pip" ]; then exit 1 fi + /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent PIP_STATUS="${PIPESTATUS[0]}" set -e @@ -58,12 +59,9 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then exit 1 fi - # Smart Requirements handling cp requirements.txt .heroku/python/requirements-declared.txt /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt - echo - # Install test dependencies, for CI. if [ "$INSTALL_TEST" ]; then if [[ -f "$1/requirements-test.txt" ]]; then diff --git a/bin/steps/python b/bin/steps/python index c06db37..a5c1e12 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -86,14 +86,6 @@ if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version fi -# need to clear the cache for first time installing SQLite3, -# since the version is changing and could lead to runtime errors -# with compiled extensions. -if [ -d .heroku/python ] && [ ! -f .heroku/python-sqlite3-version ] && python_sqlite3_check "$PYTHON_VERSION"; then - puts-step "Need to update SQLite3, clearing cache" - rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor -fi - if [ -f .heroku/python-version ]; then if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then puts-step "Found $(cat .heroku/python-version), removing" @@ -103,6 +95,29 @@ if [ -f .heroku/python-version ]; then fi fi +# Check if we should reinstall python dependencies +if [[ ! -f "$CACHE_DIR/.heroku/requirements.txt" ]]; then + # IF there's no cached dependencies, update cached version of requirements.txt + # This should only run for new apps and first deploys after this update + cp -R "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt" + # If we don't already have a python version, this is a new app + if [ -f .heroku/python-version ]; then + puts-step "Clearing cached dependencies" + # if there are any differences, clear the Python cache + # Installing Python over again does not take noticably more time + rm -rf .heroku/python + unset SKIP_INSTALL + fi +else + # IF there IS a cached directory, check for differences with the new one + if ! diff "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt"; then + puts-step "Clearing cached dependencies" + # if there are any differences, clear the Python cache + # Installing Python over again does not take noticably more time + rm -rf .heroku/python + unset SKIP_INSTALL + fi +fi if [ ! "$SKIP_INSTALL" ]; then puts-step "Installing $PYTHON_VERSION" @@ -137,7 +152,7 @@ if ! curl -s "${GETPIP_URL}" -o "$GETPIP_PY" &> /dev/null; then exit 1 fi -# If Pip isn't up to date: +# If a new Python has been installed or Pip isn't up to date: if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then puts-step "Installing pip" diff --git a/spec/hatchet/python_spec.rb b/spec/hatchet/python_spec.rb index 98f0018..ac6c3e1 100644 --- a/spec/hatchet/python_spec.rb +++ b/spec/hatchet/python_spec.rb @@ -1,10 +1,28 @@ require_relative '../spec_helper' -describe "Python!!!!!!!!!!!" do +describe "Default Python Deploy" do it "🐍" do Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app| expect(app.output).to match(/Installing pip/) expect(app.run('python -V')).to match(/3.6.10/) + + + expect(app.output).to_not match("Clearing cached dependencies") + + # Redeploy + run!(%Q{echo "flask" >> requirements.txt}) + run!(%Q{git add . ; git commit --allow-empty -m next}) + app.push! + + # Check for the cache tohave cleared + expect(app.output).to match("Clearing cached dependencies") + + run!(%Q{git commit --allow-empty -m next}) + app.push! + + # The cache should not clear with no changes + expect(app.output).to_not match("Clearing cached dependencies") + end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 12a11b8..dc7acd1 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -20,4 +20,10 @@ if ENV['TRAVIS'] exit 0 if ENV['TRAVIS_PULL_REQUEST'] != 'false' && ENV['TRAVIS_BRANCH'] == 'master' end -DEFAULT_STACK = 'heroku-16' +DEFAULT_STACK = 'heroku-18' + +def run!(cmd) + out = `#{cmd}` + raise "Error running command #{cmd} with output: #{out}" unless $?.success? + return out +end