From 38d73effc1acf32c7f200142ea017d3ef2c665b6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 09:47:14 -0400 Subject: [PATCH 01/33] an attempt at fixing caching --- bin/compile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index d81772e..20c3289 100755 --- a/bin/compile +++ b/bin/compile @@ -20,7 +20,8 @@ CACHE_DIR=$2 ENV_DIR=$3 -CACHED_DIRS=".heroku" +CACHED_DIR=".heroku" +CACHED_DIRS=".heroku/python .heroku/python-stack .heroku/python-version .heroku/vendor .heroku/venv" # Static configurations for virtualenv caches. VIRTUALENV_LOC=".heroku/venv" @@ -145,6 +146,7 @@ bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache + mkdir -p $CACHEDIR/$CACHED_DIR for dir in $CACHED_DIRS; do cp -R $CACHE_DIR/$dir . &> /dev/null || true done @@ -209,6 +211,7 @@ bpwatch stop post_compile # Store new artifacts in cache. bpwatch start dump_cache + mkdir -p $CACHE_DIR/$CACHED_DIR for dir in $CACHED_DIRS; do rm -rf $CACHE_DIR/$dir cp -R $dir $CACHE_DIR/ From 44b2ef0c4ee82b87f4d89941ae6866a00d013b5b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 09:47:28 -0400 Subject: [PATCH 02/33] an attempt at fixing caching --- bin/compile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 20c3289..33aafce 100755 --- a/bin/compile +++ b/bin/compile @@ -214,7 +214,7 @@ bpwatch start dump_cache mkdir -p $CACHE_DIR/$CACHED_DIR for dir in $CACHED_DIRS; do rm -rf $CACHE_DIR/$dir - cp -R $dir $CACHE_DIR/ + cp -R $dir $CACHE_DIR/ 2>/dev/null done bpwatch stop dump_cache From bd90eecd8cc39c7975730dd06505a6c0a45f5758 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 09:49:49 -0400 Subject: [PATCH 03/33] fix bug in cache_dir --- bin/compile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 33aafce..6405594 100755 --- a/bin/compile +++ b/bin/compile @@ -146,7 +146,7 @@ bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache - mkdir -p $CACHEDIR/$CACHED_DIR + mkdir -p $CACHE_DIR/$CACHED_DIR for dir in $CACHED_DIRS; do cp -R $CACHE_DIR/$dir . &> /dev/null || true done From c61f907079902841109ac43a1b3b9bb725e77b5a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 09:53:47 -0400 Subject: [PATCH 04/33] echo --- bin/compile | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/compile b/bin/compile index 6405594..4632f2a 100755 --- a/bin/compile +++ b/bin/compile @@ -148,6 +148,7 @@ bpwatch stop clear_old_venvs bpwatch start restore_cache mkdir -p $CACHE_DIR/$CACHED_DIR for dir in $CACHED_DIRS; do + echo "cp -R $CACHE_DIR/$dir ." cp -R $CACHE_DIR/$dir . &> /dev/null || true done bpwatch stop restore_cache From e77090b6b8ae61ac5af1c92bbfe40d9b35b05a40 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 09:55:48 -0400 Subject: [PATCH 05/33] copy them to the cache dir --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 4632f2a..31dce06 100755 --- a/bin/compile +++ b/bin/compile @@ -148,8 +148,8 @@ bpwatch stop clear_old_venvs bpwatch start restore_cache mkdir -p $CACHE_DIR/$CACHED_DIR for dir in $CACHED_DIRS; do - echo "cp -R $CACHE_DIR/$dir ." - cp -R $CACHE_DIR/$dir . &> /dev/null || true + echo "cp -R $CACHE_DIR/$dir $CACHED_DIR" + cp -R $CACHE_DIR/$dir $CACHED_DIR &> /dev/null || true done bpwatch stop restore_cache From e6d395fa27f81b05a60be108c7e9818593e374d0 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 09:57:13 -0400 Subject: [PATCH 06/33] ls /app/.heroku --- bin/compile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/compile b/bin/compile index 31dce06..da62dcc 100755 --- a/bin/compile +++ b/bin/compile @@ -153,6 +153,8 @@ bpwatch start restore_cache done bpwatch stop restore_cache +ls /app/.heroku/ + set +e # Create set-aside `.heroku` folder. mkdir .heroku &> /dev/null From d8623ae454e43e6de735dd73517b71d464cb96a7 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:02:34 -0400 Subject: [PATCH 07/33] whitelist approach --- bin/compile | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/bin/compile b/bin/compile index da62dcc..372ad3d 100755 --- a/bin/compile +++ b/bin/compile @@ -20,8 +20,8 @@ CACHE_DIR=$2 ENV_DIR=$3 -CACHED_DIR=".heroku" -CACHED_DIRS=".heroku/python .heroku/python-stack .heroku/python-version .heroku/vendor .heroku/venv" +CACHED_DIRS=".heroku" +WHITELISTED_CACHE_ENTRIES="python|python-stack|python-version|venv|vendor" # Static configurations for virtualenv caches. VIRTUALENV_LOC=".heroku/venv" @@ -146,15 +146,12 @@ bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache - mkdir -p $CACHE_DIR/$CACHED_DIR for dir in $CACHED_DIRS; do - echo "cp -R $CACHE_DIR/$dir $CACHED_DIR" - cp -R $CACHE_DIR/$dir $CACHED_DIR &> /dev/null || true + cp -R $CACHE_DIR/$dir . &> /dev/null || true done + ls -1 $CACHED_DIRS | grep -E -v $WHITELISTED_CACHE_ENTRIES | xargs rm -f bpwatch stop restore_cache -ls /app/.heroku/ - set +e # Create set-aside `.heroku` folder. mkdir .heroku &> /dev/null @@ -214,10 +211,9 @@ bpwatch stop post_compile # Store new artifacts in cache. bpwatch start dump_cache - mkdir -p $CACHE_DIR/$CACHED_DIR for dir in $CACHED_DIRS; do rm -rf $CACHE_DIR/$dir - cp -R $dir $CACHE_DIR/ 2>/dev/null + cp -R $dir $CACHE_DIR/ done bpwatch stop dump_cache From 6ab397db1016811d28572a49060e736c12f3f2c2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:03:50 -0400 Subject: [PATCH 08/33] whitelist --- bin/compile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 372ad3d..0b62cc4 100755 --- a/bin/compile +++ b/bin/compile @@ -149,7 +149,7 @@ bpwatch start restore_cache for dir in $CACHED_DIRS; do cp -R $CACHE_DIR/$dir . &> /dev/null || true done - ls -1 $CACHED_DIRS | grep -E -v $WHITELISTED_CACHE_ENTRIES | xargs rm -f + ls -1 | grep -E -v "$WHITELISTED_CACHE_ENTRIES" | xargs rm -f bpwatch stop restore_cache set +e From c0fbb0723aa2c9fbf8cb89018729a839c3a2a722 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:13:09 -0400 Subject: [PATCH 09/33] attempt to whitelist cache entries for python buildpack --- bin/compile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bin/compile b/bin/compile index 0b62cc4..d7b639c 100755 --- a/bin/compile +++ b/bin/compile @@ -6,6 +6,7 @@ # Fail fast and fail hard. set -eo pipefail +shopt -s extglob [ "$BUILDPACK_XTRACE" ] && set -o xtrace @@ -20,7 +21,7 @@ CACHE_DIR=$2 ENV_DIR=$3 -CACHED_DIRS=".heroku" +CACHED_DIR=".heroku" WHITELISTED_CACHE_ENTRIES="python|python-stack|python-version|venv|vendor" # Static configurations for virtualenv caches. @@ -144,12 +145,14 @@ bpwatch start clear_old_venvs [ -d $CACHE_DIR/$VIRTUALENV_LOC ] && rm -fr $CACHE_DIR/.heroku/venv $CACHE_DIR/.heroku/src bpwatch stop clear_old_venvs +cd $CACHE_DIR +rm !($WHITELISTED_CACHE_ENTRIES) + +cd $BUILD_DIR + # Restore old artifacts from the cache. bpwatch start restore_cache - for dir in $CACHED_DIRS; do - cp -R $CACHE_DIR/$dir . &> /dev/null || true - done - ls -1 | grep -E -v "$WHITELISTED_CACHE_ENTRIES" | xargs rm -f + cp -R $CACHE_DIR/$CACHED_DIR . &> /dev/null || true bpwatch stop restore_cache set +e @@ -211,10 +214,8 @@ bpwatch stop post_compile # Store new artifacts in cache. bpwatch start dump_cache - for dir in $CACHED_DIRS; do rm -rf $CACHE_DIR/$dir cp -R $dir $CACHE_DIR/ - done bpwatch stop dump_cache # ### Fin. From 24cc273800a1a8a05cf3aa51ffc63e8810daa42a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:13:23 -0400 Subject: [PATCH 10/33] whitelist cache --- bin/compile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/compile b/bin/compile index d7b639c..69ea53f 100755 --- a/bin/compile +++ b/bin/compile @@ -145,16 +145,16 @@ bpwatch start clear_old_venvs [ -d $CACHE_DIR/$VIRTUALENV_LOC ] && rm -fr $CACHE_DIR/.heroku/venv $CACHE_DIR/.heroku/src bpwatch stop clear_old_venvs -cd $CACHE_DIR -rm !($WHITELISTED_CACHE_ENTRIES) - -cd $BUILD_DIR - # Restore old artifacts from the cache. bpwatch start restore_cache cp -R $CACHE_DIR/$CACHED_DIR . &> /dev/null || true bpwatch stop restore_cache +# Cleanup what's in the cache. +cd $CACHED_DIR +rm !($WHITELISTED_CACHE_ENTRIES) +cd $BUILD_DIR + set +e # Create set-aside `.heroku` folder. mkdir .heroku &> /dev/null From 74af94132ad1e47cffdf7f6cebbcfc7f2cde7e51 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:14:54 -0400 Subject: [PATCH 11/33] try hardcoding it in --- bin/compile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 69ea53f..868d355 100755 --- a/bin/compile +++ b/bin/compile @@ -152,7 +152,7 @@ bpwatch stop restore_cache # Cleanup what's in the cache. cd $CACHED_DIR -rm !($WHITELISTED_CACHE_ENTRIES) +rm !(python|python-stack|python-version|venv|vendor) cd $BUILD_DIR set +e From b46cc0c6da96aa23e7bc7f079e23047e055db58c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:16:10 -0400 Subject: [PATCH 12/33] ignore cleanup the cache --- bin/compile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bin/compile b/bin/compile index 868d355..babd780 100755 --- a/bin/compile +++ b/bin/compile @@ -150,11 +150,6 @@ bpwatch start restore_cache cp -R $CACHE_DIR/$CACHED_DIR . &> /dev/null || true bpwatch stop restore_cache -# Cleanup what's in the cache. -cd $CACHED_DIR -rm !(python|python-stack|python-version|venv|vendor) -cd $BUILD_DIR - set +e # Create set-aside `.heroku` folder. mkdir .heroku &> /dev/null From beb8c705858635e688a279dd2d6d0b6b233b6a92 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:17:29 -0400 Subject: [PATCH 13/33] cleanup --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index babd780..da62406 100755 --- a/bin/compile +++ b/bin/compile @@ -209,8 +209,8 @@ bpwatch stop post_compile # Store new artifacts in cache. bpwatch start dump_cache - rm -rf $CACHE_DIR/$dir - cp -R $dir $CACHE_DIR/ + rm -rf $CACHE_DIR/$CACHED_DIR + cp -R $CACHED_DIR $CACHE_DIR/ bpwatch stop dump_cache # ### Fin. From 21dd1782fa1ab4b6300974b80133bd587058303f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:25:49 -0400 Subject: [PATCH 14/33] whitelisting --- bin/compile | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/bin/compile b/bin/compile index da62406..ee70477 100755 --- a/bin/compile +++ b/bin/compile @@ -207,26 +207,33 @@ bpwatch start post_compile source $BIN_DIR/steps/hooks/post_compile bpwatch stop post_compile -# Store new artifacts in cache. -bpwatch start dump_cache - rm -rf $CACHE_DIR/$CACHED_DIR - cp -R $CACHED_DIR $CACHE_DIR/ -bpwatch stop dump_cache # ### Fin. if [[ ! "$DOCKER_BUILD" ]]; then bpwatch start appdir_commit deep-rm $ORIG_BUILD_DIR - deep-mv $BUILD_DIR $ORIG_BUILD_DIR + deep-cp $BUILD_DIR $ORIG_BUILD_DIR bpwatch stop appdir_commit bpwatch start anvil_appdir_commit if [ "$SLUG_ID" ]; then - deep-mv $TMP_APP_DIR $APP_DIR + deep-cp $TMP_APP_DIR $APP_DIR fi bpwatch stop anvil_appdir_commit bpwatch stop compile fi + +# Remove all non-whitelisted objects from cache. +cd $CACHED_DIR +find . -type f -not \( -name 'python' -or -name 'python-version' -or -name 'python-stack' -or -name 'vendor' -or -name 'venv'\) -delete +cd $BUILD_DIR + +# Store new artifacts in cache. +bpwatch start dump_cache + rm -rf $CACHE_DIR/$CACHED_DIR + cp -R $CACHED_DIR $CACHE_DIR/ +bpwatch stop dump_cache + From 67063fc34fb80d05b5cdf881b53cba6b596dd38c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:27:03 -0400 Subject: [PATCH 15/33] fix --- bin/compile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index ee70477..e5bb4a7 100755 --- a/bin/compile +++ b/bin/compile @@ -228,7 +228,7 @@ fi # Remove all non-whitelisted objects from cache. cd $CACHED_DIR -find . -type f -not \( -name 'python' -or -name 'python-version' -or -name 'python-stack' -or -name 'vendor' -or -name 'venv'\) -delete +find . -type f -not \( -name 'python' -or -name 'python-version' -or -name 'python-stack' -or -name 'vendor' -or -name 'venv' \) -delete cd $BUILD_DIR # Store new artifacts in cache. From 921a4c31a800093289459ed4016667b06c4d7d0e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:31:39 -0400 Subject: [PATCH 16/33] back to deep mv --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index e5bb4a7..1764e23 100755 --- a/bin/compile +++ b/bin/compile @@ -213,12 +213,12 @@ if [[ ! "$DOCKER_BUILD" ]]; then bpwatch start appdir_commit deep-rm $ORIG_BUILD_DIR - deep-cp $BUILD_DIR $ORIG_BUILD_DIR + deep-mv $BUILD_DIR $ORIG_BUILD_DIR bpwatch stop appdir_commit bpwatch start anvil_appdir_commit if [ "$SLUG_ID" ]; then - deep-cp $TMP_APP_DIR $APP_DIR + deep-mv $TMP_APP_DIR $APP_DIR fi bpwatch stop anvil_appdir_commit From 7b9bc848ee1ecda999ea9704ea0c0c3f8218f937 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:33:55 -0400 Subject: [PATCH 17/33] starting over --- bin/compile | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/bin/compile b/bin/compile index 1764e23..d81772e 100755 --- a/bin/compile +++ b/bin/compile @@ -6,7 +6,6 @@ # Fail fast and fail hard. set -eo pipefail -shopt -s extglob [ "$BUILDPACK_XTRACE" ] && set -o xtrace @@ -21,8 +20,7 @@ CACHE_DIR=$2 ENV_DIR=$3 -CACHED_DIR=".heroku" -WHITELISTED_CACHE_ENTRIES="python|python-stack|python-version|venv|vendor" +CACHED_DIRS=".heroku" # Static configurations for virtualenv caches. VIRTUALENV_LOC=".heroku/venv" @@ -147,7 +145,9 @@ bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache - cp -R $CACHE_DIR/$CACHED_DIR . &> /dev/null || true + for dir in $CACHED_DIRS; do + cp -R $CACHE_DIR/$dir . &> /dev/null || true + done bpwatch stop restore_cache set +e @@ -207,6 +207,13 @@ bpwatch start post_compile source $BIN_DIR/steps/hooks/post_compile bpwatch stop post_compile +# Store new artifacts in cache. +bpwatch start dump_cache + for dir in $CACHED_DIRS; do + rm -rf $CACHE_DIR/$dir + cp -R $dir $CACHE_DIR/ + done +bpwatch stop dump_cache # ### Fin. if [[ ! "$DOCKER_BUILD" ]]; then @@ -225,15 +232,3 @@ if [[ ! "$DOCKER_BUILD" ]]; then bpwatch stop compile fi - -# Remove all non-whitelisted objects from cache. -cd $CACHED_DIR -find . -type f -not \( -name 'python' -or -name 'python-version' -or -name 'python-stack' -or -name 'vendor' -or -name 'venv' \) -delete -cd $BUILD_DIR - -# Store new artifacts in cache. -bpwatch start dump_cache - rm -rf $CACHE_DIR/$CACHED_DIR - cp -R $CACHED_DIR $CACHE_DIR/ -bpwatch stop dump_cache - From c5972cdb745a3d8203079c99a492287987b8a7a7 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:37:00 -0400 Subject: [PATCH 18/33] .heroku --- bin/compile | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/bin/compile b/bin/compile index d81772e..806a829 100755 --- a/bin/compile +++ b/bin/compile @@ -20,7 +20,7 @@ CACHE_DIR=$2 ENV_DIR=$3 -CACHED_DIRS=".heroku" +CACHED_DIR=".heroku" # Static configurations for virtualenv caches. VIRTUALENV_LOC=".heroku/venv" @@ -145,9 +145,7 @@ bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache - for dir in $CACHED_DIRS; do - cp -R $CACHE_DIR/$dir . &> /dev/null || true - done + cp -R $CACHE_DIR/$CACHED_DIR . &> /dev/null || true bpwatch stop restore_cache set +e @@ -209,10 +207,8 @@ bpwatch stop post_compile # Store new artifacts in cache. bpwatch start dump_cache - for dir in $CACHED_DIRS; do - rm -rf $CACHE_DIR/$dir - cp -R $dir $CACHE_DIR/ - done + rm -rf $CACHE_DIR/$CACHED_DIR + cp -R $CACHED_DIR $CACHE_DIR/ bpwatch stop dump_cache # ### Fin. From 8b3e99adb3d1d79fe48877fe88fec1aacf00da68 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:42:57 -0400 Subject: [PATCH 19/33] manual cache population --- bin/compile | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 806a829..9314288 100755 --- a/bin/compile +++ b/bin/compile @@ -207,8 +207,22 @@ bpwatch stop post_compile # Store new artifacts in cache. bpwatch start dump_cache - rm -rf $CACHE_DIR/$CACHED_DIR - cp -R $CACHED_DIR $CACHE_DIR/ + # python vendor python-stack python-version venv + rm -rf $CACHE_DIR/$CACHED_DIR/python + cp -R $CACHED_DIR/python $CACHE_DIR/python + + rm -rf $CACHE_DIR/$CACHED_DIR/python-stack + cp -R $CACHED_DIR/python-stack $CACHE_DIR/python-stack + + rm -rf $CACHE_DIR/$CACHED_DIR/python-version + cp -R $CACHED_DIR/python-version $CACHE_DIR/python-version + + rm -rf $CACHE_DIR/$CACHED_DIR/vendor + cp -R $CACHED_DIR/vendor $CACHE_DIR/vendor + + rm -rf $CACHE_DIR/$CACHED_DIR/venv + cp -R $CACHED_DIR/venv $CACHE_DIR/venv + bpwatch stop dump_cache # ### Fin. From 4513dd75220614e07c3474693b6180b6950caca0 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:47:09 -0400 Subject: [PATCH 20/33] don't error out on directories that may not exist --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 9314288..d2e665e 100755 --- a/bin/compile +++ b/bin/compile @@ -218,10 +218,10 @@ bpwatch start dump_cache cp -R $CACHED_DIR/python-version $CACHE_DIR/python-version rm -rf $CACHE_DIR/$CACHED_DIR/vendor - cp -R $CACHED_DIR/vendor $CACHE_DIR/vendor + cp -R $CACHED_DIR/vendor $CACHE_DIR/vendor 2>/dev/null rm -rf $CACHE_DIR/$CACHED_DIR/venv - cp -R $CACHED_DIR/venv $CACHE_DIR/venv + cp -R $CACHED_DIR/venv $CACHE_DIR/venv 2>/dev/null bpwatch stop dump_cache From 69e9368c2d615fdda46e27a86041f2c038f17708 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:48:50 -0400 Subject: [PATCH 21/33] further don't error out --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index d2e665e..52296f5 100755 --- a/bin/compile +++ b/bin/compile @@ -218,10 +218,10 @@ bpwatch start dump_cache cp -R $CACHED_DIR/python-version $CACHE_DIR/python-version rm -rf $CACHE_DIR/$CACHED_DIR/vendor - cp -R $CACHED_DIR/vendor $CACHE_DIR/vendor 2>/dev/null + cp -R $CACHED_DIR/vendor $CACHE_DIR/vendor &> /dev/null || true rm -rf $CACHE_DIR/$CACHED_DIR/venv - cp -R $CACHED_DIR/venv $CACHE_DIR/venv 2>/dev/null + cp -R $CACHED_DIR/venv $CACHE_DIR/venv &> /dev/null || true bpwatch stop dump_cache From 9f666cee7991653f823d58c3aad3c0d01a718311 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 10:57:51 -0400 Subject: [PATCH 22/33] explicit is better than implicit --- bin/compile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 52296f5..87faaa3 100755 --- a/bin/compile +++ b/bin/compile @@ -145,7 +145,12 @@ bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache - cp -R $CACHE_DIR/$CACHED_DIR . &> /dev/null || true + mkdir -p $CACHED_DIR + cp -R $CACHE_DIR/$CACHED_DIR/python $CACHE_DIR/python &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/python-stack $CACHE_DIR/python-stack &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/python-version $CACHE_DIR/python-version &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/vendor $CACHE_DIR/vendor &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/venv $CACHE_DIR/venv &> /dev/null || true bpwatch stop restore_cache set +e From 5e212e4db89dc20e459998ebf4d7f631ac78d083 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 11:01:51 -0400 Subject: [PATCH 23/33] CACHED_DIR --- bin/compile | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/bin/compile b/bin/compile index 87faaa3..9eb65da 100755 --- a/bin/compile +++ b/bin/compile @@ -145,12 +145,17 @@ bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache - mkdir -p $CACHED_DIR - cp -R $CACHE_DIR/$CACHED_DIR/python $CACHE_DIR/python &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/python-stack $CACHE_DIR/python-stack &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/python-version $CACHE_DIR/python-version &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/vendor $CACHE_DIR/vendor &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/venv $CACHE_DIR/venv &> /dev/null || true + # cp -R $CACHE_DIR/$CACHED_DIR . &> /dev/null || true + + # python vendor python-stack python-version venv + cp -R $CACHE_DIR/$CACHED_DIR/python $CACHED_DIR/python + cp -R $CACHE_DIR/$CACHED_DIR/python-stack $CACHED_DIR/python-stack + cp -R $CACHE_DIR/$CACHED_DIR/python-version $CACHED_DIR/python-version + cp -R $CACHE_DIR/$CACHED_DIR/vendor $CACHED_DIR/vendor &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/venv $CACHED_DIR/venv &> /dev/null || true + + + bpwatch stop restore_cache set +e From c78bf770559d7155f2c0bc4cb09eb8159e61d4b6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 11:02:17 -0400 Subject: [PATCH 24/33] allow them to fail --- bin/compile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/compile b/bin/compile index 9eb65da..8b7b5e3 100755 --- a/bin/compile +++ b/bin/compile @@ -148,9 +148,9 @@ bpwatch start restore_cache # cp -R $CACHE_DIR/$CACHED_DIR . &> /dev/null || true # python vendor python-stack python-version venv - cp -R $CACHE_DIR/$CACHED_DIR/python $CACHED_DIR/python - cp -R $CACHE_DIR/$CACHED_DIR/python-stack $CACHED_DIR/python-stack - cp -R $CACHE_DIR/$CACHED_DIR/python-version $CACHED_DIR/python-version + cp -R $CACHE_DIR/$CACHED_DIR/python $CACHED_DIR/python &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/python-stack $CACHED_DIR/python-stack &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/python-version $CACHED_DIR/python-version &> /dev/null || true cp -R $CACHE_DIR/$CACHED_DIR/vendor $CACHED_DIR/vendor &> /dev/null || true cp -R $CACHE_DIR/$CACHED_DIR/venv $CACHED_DIR/venv &> /dev/null || true From fd4ab23f5027adc00d9506f54f6624feea2dfc8d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 11:03:49 -0400 Subject: [PATCH 25/33] debug --- bin/compile | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/compile b/bin/compile index 8b7b5e3..b06e199 100755 --- a/bin/compile +++ b/bin/compile @@ -147,6 +147,7 @@ bpwatch stop clear_old_venvs bpwatch start restore_cache # cp -R $CACHE_DIR/$CACHED_DIR . &> /dev/null || true + ls -a $CACHE_DIR # python vendor python-stack python-version venv cp -R $CACHE_DIR/$CACHED_DIR/python $CACHED_DIR/python &> /dev/null || true cp -R $CACHE_DIR/$CACHED_DIR/python-stack $CACHED_DIR/python-stack &> /dev/null || true From 23e1164c9369919245b385202533212025e27fa7 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 11:09:39 -0400 Subject: [PATCH 26/33] try removing .heroku --- bin/compile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/compile b/bin/compile index b06e199..5faa8c3 100755 --- a/bin/compile +++ b/bin/compile @@ -149,11 +149,11 @@ bpwatch start restore_cache ls -a $CACHE_DIR # python vendor python-stack python-version venv - cp -R $CACHE_DIR/$CACHED_DIR/python $CACHED_DIR/python &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/python-stack $CACHED_DIR/python-stack &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/python-version $CACHED_DIR/python-version &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/vendor $CACHED_DIR/vendor &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/venv $CACHED_DIR/venv &> /dev/null || true + cp -R $CACHE_DIR/python $CACHED_DIR/python &> /dev/null || true + cp -R $CACHE_DIR/python-stack $CACHED_DIR/python-stack &> /dev/null || true + cp -R $CACHE_DIR/python-version $CACHED_DIR/python-version &> /dev/null || true + cp -R $CACHE_DIR/vendor $CACHED_DIR/vendor &> /dev/null || true + cp -R $CACHE_DIR/venv $CACHED_DIR/venv &> /dev/null || true From f7387427c85da2433d0e8e37d5d39bb9939fd102 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 11:12:55 -0400 Subject: [PATCH 27/33] changes --- bin/compile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bin/compile b/bin/compile index 5faa8c3..3888dc9 100755 --- a/bin/compile +++ b/bin/compile @@ -149,11 +149,11 @@ bpwatch start restore_cache ls -a $CACHE_DIR # python vendor python-stack python-version venv - cp -R $CACHE_DIR/python $CACHED_DIR/python &> /dev/null || true - cp -R $CACHE_DIR/python-stack $CACHED_DIR/python-stack &> /dev/null || true - cp -R $CACHE_DIR/python-version $CACHED_DIR/python-version &> /dev/null || true - cp -R $CACHE_DIR/vendor $CACHED_DIR/vendor &> /dev/null || true - cp -R $CACHE_DIR/venv $CACHED_DIR/venv &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/python $CACHED_DIR/python &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/python-stack $CACHED_DIR/python-stack &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/python-version $CACHED_DIR/python-version &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/vendor $CACHED_DIR/vendor &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/venv $CACHED_DIR/venv &> /dev/null || true @@ -220,19 +220,19 @@ bpwatch stop post_compile bpwatch start dump_cache # python vendor python-stack python-version venv rm -rf $CACHE_DIR/$CACHED_DIR/python - cp -R $CACHED_DIR/python $CACHE_DIR/python + cp -R $CACHED_DIR/python $CACHE_DIR/$CACHED_DIR/python rm -rf $CACHE_DIR/$CACHED_DIR/python-stack - cp -R $CACHED_DIR/python-stack $CACHE_DIR/python-stack + cp -R $CACHED_DIR/python-stack $CACHE_DIR/$CACHED_DIR/python-stack rm -rf $CACHE_DIR/$CACHED_DIR/python-version - cp -R $CACHED_DIR/python-version $CACHE_DIR/python-version + cp -R $CACHED_DIR/python-version $CACHE_DIR/$CACHED_DIR/python-version rm -rf $CACHE_DIR/$CACHED_DIR/vendor - cp -R $CACHED_DIR/vendor $CACHE_DIR/vendor &> /dev/null || true + cp -R $CACHED_DIR/vendor $CACHE_DIR/$CACHED_DIR/vendor &> /dev/null || true rm -rf $CACHE_DIR/$CACHED_DIR/venv - cp -R $CACHED_DIR/venv $CACHE_DIR/venv &> /dev/null || true + cp -R $CACHED_DIR/venv $CACHE_DIR/$CACHED_DIR/venv &> /dev/null || true bpwatch stop dump_cache From eef41088b15e14abc41f484d2e9bc76038fe1a45 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Oct 2015 11:23:34 -0400 Subject: [PATCH 28/33] mkdir-p --- bin/compile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/compile b/bin/compile index 3888dc9..5ac57ec 100755 --- a/bin/compile +++ b/bin/compile @@ -218,6 +218,8 @@ bpwatch stop post_compile # Store new artifacts in cache. bpwatch start dump_cache + mkdir -p $CACHE_DIR/$CACHED_DIR + # python vendor python-stack python-version venv rm -rf $CACHE_DIR/$CACHED_DIR/python cp -R $CACHED_DIR/python $CACHE_DIR/$CACHED_DIR/python @@ -233,6 +235,8 @@ bpwatch start dump_cache rm -rf $CACHE_DIR/$CACHED_DIR/venv cp -R $CACHED_DIR/venv $CACHE_DIR/$CACHED_DIR/venv &> /dev/null || true + mkdir -p $CACHE_DIR/$CACHED_DIR + bpwatch stop dump_cache From d96914ab2e6003e682e66250a46a5b3b75082f3f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 15 Oct 2015 11:40:55 -0400 Subject: [PATCH 29/33] restore master's compile script --- bin/compile | 41 ++++++++--------------------------------- 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/bin/compile b/bin/compile index 5ac57ec..d81772e 100755 --- a/bin/compile +++ b/bin/compile @@ -20,7 +20,7 @@ CACHE_DIR=$2 ENV_DIR=$3 -CACHED_DIR=".heroku" +CACHED_DIRS=".heroku" # Static configurations for virtualenv caches. VIRTUALENV_LOC=".heroku/venv" @@ -145,18 +145,9 @@ bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache - # cp -R $CACHE_DIR/$CACHED_DIR . &> /dev/null || true - - ls -a $CACHE_DIR - # python vendor python-stack python-version venv - cp -R $CACHE_DIR/$CACHED_DIR/python $CACHED_DIR/python &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/python-stack $CACHED_DIR/python-stack &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/python-version $CACHED_DIR/python-version &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/vendor $CACHED_DIR/vendor &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/venv $CACHED_DIR/venv &> /dev/null || true - - - + for dir in $CACHED_DIRS; do + cp -R $CACHE_DIR/$dir . &> /dev/null || true + done bpwatch stop restore_cache set +e @@ -218,26 +209,10 @@ bpwatch stop post_compile # Store new artifacts in cache. bpwatch start dump_cache - mkdir -p $CACHE_DIR/$CACHED_DIR - - # python vendor python-stack python-version venv - rm -rf $CACHE_DIR/$CACHED_DIR/python - cp -R $CACHED_DIR/python $CACHE_DIR/$CACHED_DIR/python - - rm -rf $CACHE_DIR/$CACHED_DIR/python-stack - cp -R $CACHED_DIR/python-stack $CACHE_DIR/$CACHED_DIR/python-stack - - rm -rf $CACHE_DIR/$CACHED_DIR/python-version - cp -R $CACHED_DIR/python-version $CACHE_DIR/$CACHED_DIR/python-version - - rm -rf $CACHE_DIR/$CACHED_DIR/vendor - cp -R $CACHED_DIR/vendor $CACHE_DIR/$CACHED_DIR/vendor &> /dev/null || true - - rm -rf $CACHE_DIR/$CACHED_DIR/venv - cp -R $CACHED_DIR/venv $CACHE_DIR/$CACHED_DIR/venv &> /dev/null || true - mkdir -p $CACHE_DIR/$CACHED_DIR - - + for dir in $CACHED_DIRS; do + rm -rf $CACHE_DIR/$dir + cp -R $dir $CACHE_DIR/ + done bpwatch stop dump_cache # ### Fin. From 05edd6b065aa7fab89f3312102cc1d5032258e2f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 27 Oct 2015 12:15:53 -0400 Subject: [PATCH 30/33] update to caching mechanism --- bin/compile | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bin/compile b/bin/compile index d81772e..a62aa1b 100755 --- a/bin/compile +++ b/bin/compile @@ -20,7 +20,7 @@ CACHE_DIR=$2 ENV_DIR=$3 -CACHED_DIRS=".heroku" +CACHED_DIR=".heroku" # Static configurations for virtualenv caches. VIRTUALENV_LOC=".heroku/venv" @@ -145,9 +145,14 @@ bpwatch stop clear_old_venvs # Restore old artifacts from the cache. bpwatch start restore_cache - for dir in $CACHED_DIRS; do - cp -R $CACHE_DIR/$dir . &> /dev/null || true - done + mkdir -p .heroku + + cp -R $CACHE_DIR/$CACHED_DIR/python .heroku/ &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/python-stack .heroku/ &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/python-version .heroku/ &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/vendor .heroku/ &> /dev/null || true + cp -R $CACHE_DIR/$CACHED_DIR/venv .heroku/ &> /dev/null || true + bpwatch stop restore_cache set +e @@ -209,10 +214,10 @@ bpwatch stop post_compile # Store new artifacts in cache. bpwatch start dump_cache - for dir in $CACHED_DIRS; do - rm -rf $CACHE_DIR/$dir - cp -R $dir $CACHE_DIR/ - done + + rm -rf $CACHE_DIR/$CACHED_DIR + cp -R $CACHED_DIR $CACHE_DIR/ + bpwatch stop dump_cache # ### Fin. From d5fed79e865e8aa5c437d66a6c83a457c3ad2788 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 27 Oct 2015 12:22:40 -0400 Subject: [PATCH 31/33] update to caching mechanism pt 2 --- bin/compile | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index a62aa1b..58a4ffd 100755 --- a/bin/compile +++ b/bin/compile @@ -215,8 +215,18 @@ bpwatch stop post_compile # Store new artifacts in cache. bpwatch start dump_cache - rm -rf $CACHE_DIR/$CACHED_DIR - cp -R $CACHED_DIR $CACHE_DIR/ + rm -rf $CACHE_DIR/.heroku/python + rm -rf $CACHE_DIR/.heroku/python-version + rm -rf $CACHE_DIR/.heroku/python-stack + rm -rf $CACHE_DIR/.heroku/vendor + rm -rf $CACHE_DIR/.heroku/venv + + mkdir -p $CACHE_DIR/.heroku + cp -R .heroku/python $CACHE_DIR/.heroku + cp -R .heroku/python-version $CACHE_DIR/.heroku + cp -R .heroku/python-stack $CACHE_DIR/.heroku + cp -R .heroku/vendor $CACHE_DIR/.heroku &> /dev/null || true + cp -R .heroku/venv $CACHE_DIR/.heroku &> /dev/null || true bpwatch stop dump_cache From c9acc4154b8587e5a1d4bbd00b1f331df624c0c5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 27 Oct 2015 12:23:15 -0400 Subject: [PATCH 32/33] / --- bin/compile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/compile b/bin/compile index 58a4ffd..235f77b 100755 --- a/bin/compile +++ b/bin/compile @@ -222,11 +222,11 @@ bpwatch start dump_cache rm -rf $CACHE_DIR/.heroku/venv mkdir -p $CACHE_DIR/.heroku - cp -R .heroku/python $CACHE_DIR/.heroku - cp -R .heroku/python-version $CACHE_DIR/.heroku - cp -R .heroku/python-stack $CACHE_DIR/.heroku - cp -R .heroku/vendor $CACHE_DIR/.heroku &> /dev/null || true - cp -R .heroku/venv $CACHE_DIR/.heroku &> /dev/null || true + cp -R .heroku/python $CACHE_DIR/.heroku/ + cp -R .heroku/python-version $CACHE_DIR/.heroku/ + cp -R .heroku/python-stack $CACHE_DIR/.heroku/ + cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true + cp -R .heroku/venv $CACHE_DIR/.heroku/ &> /dev/null || true bpwatch stop dump_cache From 6b3f63252fdb44d19700985821c4879fbf16a2c6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 27 Oct 2015 12:50:17 -0400 Subject: [PATCH 33/33] cleanup --- bin/compile | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/bin/compile b/bin/compile index 235f77b..a748701 100755 --- a/bin/compile +++ b/bin/compile @@ -19,9 +19,6 @@ BUILD_DIR=$1 CACHE_DIR=$2 ENV_DIR=$3 - -CACHED_DIR=".heroku" - # Static configurations for virtualenv caches. VIRTUALENV_LOC=".heroku/venv" LEGACY_TRIGGER="lib/python2.7" @@ -147,11 +144,11 @@ bpwatch stop clear_old_venvs bpwatch start restore_cache mkdir -p .heroku - cp -R $CACHE_DIR/$CACHED_DIR/python .heroku/ &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/python-stack .heroku/ &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/python-version .heroku/ &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/vendor .heroku/ &> /dev/null || true - cp -R $CACHE_DIR/$CACHED_DIR/venv .heroku/ &> /dev/null || true + cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true + cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true + cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true + cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true + cp -R $CACHE_DIR/.heroku/venv .heroku/ &> /dev/null || true bpwatch stop restore_cache