From dd707c21e3998af3d6b40a2ec9e0124845ca71a2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 15:40:35 -0500 Subject: [PATCH 01/13] re-write egg-links --- bin/compile | 8 +------- bin/steps/pip-install | 3 +++ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/bin/compile b/bin/compile index 0ef5bc6..b106107 100755 --- a/bin/compile +++ b/bin/compile @@ -146,16 +146,13 @@ bpwatch stop restore_cache mkdir -p $(dirname $PROFILE_PATH) -# Make the directory for -e pip installations. -mkdir -p /app/.heroku/src - if [[ $BUILD_DIR != '/app' ]]; then # python expects to reside in /app, so set up symlinks # we will not remove these later so subsequent buildpacks can still invoke it ln -s $BUILD_DIR/.heroku/python /app/.heroku/python ln -s $BUILD_DIR/.heroku/vendor /app/.heroku/vendor ln -s $BUILD_DIR/.heroku/venv /app/.heroku/venv - # NOTE: /app/.heroku/src also exists, but is copied manually later. + ln -s $BUILD_DIR/.heroku/src /app/.heroku/src fi # Install Python. @@ -202,9 +199,6 @@ set-default-env PYTHONPATH /app/ # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS. cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH -# Deep copy the directory for -e pip installations -deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src - # Experimental post_compile hook. bpwatch start post_compile source $BIN_DIR/steps/hooks/post_compile diff --git a/bin/steps/pip-install b/bin/steps/pip-install index 7dea3ce..4bdbe1d 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -20,6 +20,9 @@ fi cp requirements.txt .heroku/python/requirements-declared.txt /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt +# Replace egg-links with new paths for /app. +find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" | xargs -n 1 sed -i -e 's#$(pwd)/#/app/#' + [ ! "$FRESH_PYTHON" ] && bpwatch stop pip_install [ "$FRESH_PYTHON" ] && bpwatch stop pip_install_first From 7f4273f47f65cf584b0e22c17cb9cf2bec0eb873 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 15:45:20 -0500 Subject: [PATCH 02/13] new replace script --- bin/steps/pip-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/steps/pip-install b/bin/steps/pip-install index 4bdbe1d..4937dff 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -21,7 +21,7 @@ cp requirements.txt .heroku/python/requirements-declared.txt /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt # Replace egg-links with new paths for /app. -find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" | xargs -n 1 sed -i -e 's#$(pwd)/#/app/#' +find .heroku/python/lib/python*/site-packages/-name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#/app/#" [ ! "$FRESH_PYTHON" ] && bpwatch stop pip_install [ "$FRESH_PYTHON" ] && bpwatch stop pip_install_first From 9c222a9350b622ac0f03341eef2b8701b3a847b5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 15:53:04 -0500 Subject: [PATCH 03/13] copy src --- bin/compile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index b106107..0ef5bc6 100755 --- a/bin/compile +++ b/bin/compile @@ -146,13 +146,16 @@ bpwatch stop restore_cache mkdir -p $(dirname $PROFILE_PATH) +# Make the directory for -e pip installations. +mkdir -p /app/.heroku/src + if [[ $BUILD_DIR != '/app' ]]; then # python expects to reside in /app, so set up symlinks # we will not remove these later so subsequent buildpacks can still invoke it ln -s $BUILD_DIR/.heroku/python /app/.heroku/python ln -s $BUILD_DIR/.heroku/vendor /app/.heroku/vendor ln -s $BUILD_DIR/.heroku/venv /app/.heroku/venv - ln -s $BUILD_DIR/.heroku/src /app/.heroku/src + # NOTE: /app/.heroku/src also exists, but is copied manually later. fi # Install Python. @@ -199,6 +202,9 @@ set-default-env PYTHONPATH /app/ # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS. cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH +# Deep copy the directory for -e pip installations +deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src + # Experimental post_compile hook. bpwatch start post_compile source $BIN_DIR/steps/hooks/post_compile From 66d754978e101ab9e915aaa32c6f23f9a8c09b3c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 15:55:04 -0500 Subject: [PATCH 04/13] fix xargs --- bin/steps/pip-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/steps/pip-install b/bin/steps/pip-install index 4937dff..e0893e4 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -21,7 +21,7 @@ cp requirements.txt .heroku/python/requirements-declared.txt /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt # Replace egg-links with new paths for /app. -find .heroku/python/lib/python*/site-packages/-name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#/app/#" +find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#/app/#" [ ! "$FRESH_PYTHON" ] && bpwatch stop pip_install [ "$FRESH_PYTHON" ] && bpwatch stop pip_install_first From 7a6f1eb010d40ffdcebb99e731dedb8bd2d806c6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 16:02:24 -0500 Subject: [PATCH 05/13] find --- bin/steps/pip-install | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/steps/pip-install b/bin/steps/pip-install index e0893e4..46af937 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -22,6 +22,7 @@ cp requirements.txt .heroku/python/requirements-declared.txt # Replace egg-links with new paths for /app. find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#/app/#" +find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#./app/#/app/#" [ ! "$FRESH_PYTHON" ] && bpwatch stop pip_install [ "$FRESH_PYTHON" ] && bpwatch stop pip_install_first From 300285a92d6c12923cd11c33454e3cba57bd523d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 16:05:22 -0500 Subject: [PATCH 06/13] find --- bin/steps/pip-install | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/steps/pip-install b/bin/steps/pip-install index 46af937..b4131df 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -21,6 +21,7 @@ cp requirements.txt .heroku/python/requirements-declared.txt /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt # Replace egg-links with new paths for /app. +find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 cat find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#/app/#" find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#./app/#/app/#" From 35cabaeebc1386541f7e5520c99726edc5a3d44d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 16:09:21 -0500 Subject: [PATCH 07/13] try . --- bin/steps/pip-install | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/steps/pip-install b/bin/steps/pip-install index b4131df..5b8a129 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -22,8 +22,7 @@ cp requirements.txt .heroku/python/requirements-declared.txt # Replace egg-links with new paths for /app. find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 cat -find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#/app/#" -find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#./app/#/app/#" +find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#./app/#" [ ! "$FRESH_PYTHON" ] && bpwatch stop pip_install [ "$FRESH_PYTHON" ] && bpwatch stop pip_install_first From 6572ad3d44de2364de4cde57fc457256112b1f0b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 16:21:03 -0500 Subject: [PATCH 08/13] test --- bin/steps/pip-install | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/steps/pip-install b/bin/steps/pip-install index 5b8a129..a139a25 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -23,6 +23,7 @@ cp requirements.txt .heroku/python/requirements-declared.txt # Replace egg-links with new paths for /app. find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 cat find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#./app/#" +find .heroku/python/lib/python*/site-packages/ -name "easy-install.pth" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#/app/#" [ ! "$FRESH_PYTHON" ] && bpwatch stop pip_install [ "$FRESH_PYTHON" ] && bpwatch stop pip_install_first From 554a8bbae6412031fc22b4dfeba0238be4ad7246 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 16:26:18 -0500 Subject: [PATCH 09/13] comment out debugging line --- bin/steps/pip-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/steps/pip-install b/bin/steps/pip-install index a139a25..8043521 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -21,7 +21,7 @@ cp requirements.txt .heroku/python/requirements-declared.txt /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt # Replace egg-links with new paths for /app. -find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 cat +# find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 cat find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#./app/#" find .heroku/python/lib/python*/site-packages/ -name "easy-install.pth" -print0 | xargs -0 -n 1 sed -i -e "s#$(pwd)/#/app/#" From 1a1cedfc2153f12828f92ebc76997a172fec9451 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 16:37:03 -0500 Subject: [PATCH 10/13] symlink eggs --- bin/compile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 0ef5bc6..82a05ed 100755 --- a/bin/compile +++ b/bin/compile @@ -202,8 +202,11 @@ set-default-env PYTHONPATH /app/ # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS. cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH -# Deep copy the directory for -e pip installations -deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src +# Symlink the directory for -e pip installations +if [[ $BUILD_DIR != '/app' ]]; then + mv /app/.heroku/src $BUILD_DIR/.heroku + ln -s /app/.heroku/src $BUILD_DIR/.heroku/src +fi # Experimental post_compile hook. bpwatch start post_compile From 0a4d32c8a53e5e195c6a9adf50f89d4f853ea45f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 16:39:34 -0500 Subject: [PATCH 11/13] cache src --- bin/compile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/compile b/bin/compile index 82a05ed..dcd5991 100755 --- a/bin/compile +++ b/bin/compile @@ -221,6 +221,7 @@ bpwatch start dump_cache rm -rf $CACHE_DIR/.heroku/python-stack rm -rf $CACHE_DIR/.heroku/vendor rm -rf $CACHE_DIR/.heroku/venv + rm -rf $CACHE_DIR/.heroku/src mkdir -p $CACHE_DIR/.heroku cp -R .heroku/python $CACHE_DIR/.heroku/ @@ -228,6 +229,7 @@ bpwatch start dump_cache cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true cp -R .heroku/venv $CACHE_DIR/.heroku/ &> /dev/null || true + cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true bpwatch stop dump_cache From 555d5bd2be6feddc1c080918975ce429a4f9c68a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 16:42:33 -0500 Subject: [PATCH 12/13] cp the cache --- bin/compile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/compile b/bin/compile index dcd5991..be26cc8 100755 --- a/bin/compile +++ b/bin/compile @@ -142,6 +142,11 @@ bpwatch start restore_cache cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true cp -R $CACHE_DIR/.heroku/venv .heroku/ &> /dev/null || true + if [[ -d $CACHE_DIR/.heroku/src ]]; then + cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true + fi + + bpwatch stop restore_cache mkdir -p $(dirname $PROFILE_PATH) From f468739cfb4040908467d28e261670a65b7cda57 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2016 16:45:04 -0500 Subject: [PATCH 13/13] v89 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7eb7592..1efbd0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Python Buildpack Changelog +## v89 + +Improved cache functionality and fix egg-links regression. + ## v88 Fixed bug with editable pip installations.