Add Hatchet Regression Tests (#969)

I went through all the tests in the Ruby buildpack and cataloged what I think can be generalized between languages to prevent regressions. This PR add tests for these cases:

* Test CI deploys run tests and use the cache [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/f488bd53c7ff0b78e17c2405166cbd4a3af75ee2/spec/hatchet/ci_spec.rb#L36)]
* Test cache for regular deploys is used on repeated deploys (This was already tested on the Python buildpack, I moved it) [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/e34c583c139911d059f5627bb25125707288f053/spec/hatchet/stack_spec.rb#L21-L25)]
    * Test modifying a requirement clears the cache appropriately (This was already tested on the Python buildpack, I moved it)
* Test deploying the getting started guide works [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/424a7245e2da86845a20d58a9482bcf2a00c3a8f/spec/hatchet/getting_started_spec.rb#L5)]
* Test that all paths set by the buildpack are absolute instead of relative [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/249d3c1a4e97068f8fd016f10fa0839709d95658/spec/hatchet/rails5_spec.rb#L68])
* Test upgrading stack invalidates the cache [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/f488bd53c7ff0b78e17c2405166cbd4a3af75ee2/spec/hatchet/stack_spec.rb#L3)]
* Test that builds fail when a bad version is specified [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/249d3c1a4e97068f8fd016f10fa0839709d95658/spec/hatchet/ruby_spec.rb#L5)]


In addition to that I've also got a CNB test with `pack-build` with the getting started app, but since python isn't `cnb` capable yet I didn't add one.

.travis.yml

@@ -6,7 +6,7 @@ branches:
   - master
 
 rvm:
-- 2.4.4
+- 2.6.6
 before_script:
   - gem install bundler -v 1.16.2
 
@@ -28,7 +28,8 @@ jobs:
     name: Run Hatchet
     script:
       - bundle exec hatchet ci:setup
-      - bundle exec rspec
+      - PARALLEL_SPLIT_TEST_PROCESSES=11 bundle exec parallel_split_test spec/hatchet/
+
 env:
   matrix:
     - TESTFOLDER=test/run-deps
@@ -37,7 +38,7 @@ env:
   global:
     - HATCHET_RETRIES=3
     - IS_RUNNING_ON_CI=true
-    - HATCHET_APP_LIMIT=5
+    - HATCHET_APP_LIMIT=80
     - HATCHET_DEPLOY_STRATEGY=git
     - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
     - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=

Gemfile

@@ -4,3 +4,5 @@ gem "rspec"
 gem "heroku_hatchet"
 gem "rspec-retry"
 gem "rake"
+gem "parallel_split_test"
+

Gemfile.lock

@@ -1,21 +1,22 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.1)
+    activesupport (6.0.2.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    concurrent-ruby (1.1.3)
+      zeitwerk (~> 2.2)
+    concurrent-ruby (1.1.6)
     diff-lcs (1.3)
     erubis (2.7.0)
-    excon (0.62.0)
+    excon (0.73.0)
     heroics (0.0.25)
       erubis (~> 2.0)
       excon
       moneta
       multi_json (>= 1.9.2)
-    heroku_hatchet (4.0.6)
+    heroku_hatchet (5.0.2)
       excon (~> 0)
       minitest-retry (~> 0.1.9)
       platform-api (~> 2)
@@ -23,13 +24,17 @@ GEM
       rrrretry (~> 1)
       thor (~> 0)
       threaded (~> 0)
-    i18n (1.1.1)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    minitest (5.11.3)
+    minitest (5.14.0)
     minitest-retry (0.1.9)
       minitest (>= 5.0)
     moneta (1.0.0)
-    multi_json (1.13.1)
+    multi_json (1.14.1)
+    parallel (1.19.1)
+    parallel_split_test (0.7.0)
+      parallel (>= 0.5.13)
+      rspec (>= 3.1.0)
     platform-api (2.2.0)
       heroics (~> 0.0.25)
       moneta (~> 1.0.0)
@@ -55,17 +60,19 @@ GEM
     thor (0.20.3)
     thread_safe (0.3.6)
     threaded (0.0.4)
-    tzinfo (1.2.5)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
+    zeitwerk (2.3.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   heroku_hatchet
+  parallel_split_test
   rake
   rspec
   rspec-retry
 
 BUNDLED WITH
-   1.16.3
+   2.1.4

hatchet.json

@@ -1,5 +1,6 @@
 {
   "python": [
-    "heroku/python-getting-started"
+    "heroku/python-getting-started",
+    "sharpstone/python_default"
   ]
 }

hatchet.lock

@@ -1,3 +1,5 @@
 ---
 - - "./repos/python/python-getting-started"
-  - 443a90c58be6881583cd7ef628e3869e3c30bb98
+  - master
+- - "./repos/python/python_default"
+  - ca947f69027b2a30be5d26f9a42f25e54f4d7a1a

spec/hatchet/ci_spec.rb

@@ -0,0 +1,33 @@
+require_relative '../spec_helper'
+
+describe "Heroku CI" do
+  it "works" do
+    before_deploy = Proc.new do
+      File.open("app.json", "w+") do |f|
+        f.puts <<~EOM
+          {
+            "environments": {
+              "test": {
+                "scripts": {
+                  "test": "nosetests"
+                }
+              }
+            }
+          }
+        EOM
+      end
+
+      run!("echo nose >> requirements.txt")
+    end
+
+    Hatchet::Runner.new("python_default", before_deploy: before_deploy).run_ci do |test_run|
+      expect(test_run.output).to match("Downloading nose")
+      expect(test_run.output).to match("OK")
+
+      test_run.run_again
+
+      expect(test_run.output).to match("installing from cache")
+      expect(test_run.output).to_not match("Downloading nose")
+    end
+  end
+end

spec/hatchet/python_spec.rb

@@ -1,68 +1,72 @@
 require_relative '../spec_helper'
 
-describe "Default Python Deploy" do
+describe "Python" do
+  describe "cache" do
+    it "functions correctly" do
+      Hatchet::Runner.new("python_default").deploy do |app|
+        expect(app.output).to match(/Installing pip/)
 
-  def set_python_version(d, v)
+        expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies")
-    Dir.chdir(d) do
+        expect(app.output).to_not match("No change in requirements detected, installing from cache")
-      File.open('runtime.txt', 'w') do |f|
+        expect(app.output).to_not match("No such file or directory")
-        f.puts "python-#{v}"
+        expect(app.output).to_not match("cp: cannot create regular file")
+
+        # Redeploy with changed requirements file
+        run!(%Q{echo "" >> requirements.txt})
+        run!(%Q{echo "pygments" >> requirements.txt})
+        run!(%Q{git add . ; git commit --allow-empty -m next})
+        app.push!
+
+        # Check the cache to have cleared
+        expect(app.output).to match("Requirements file has been changed, clearing cached dependencies")
+        expect(app.output).to_not match("No dependencies found, preparing to install")
+        expect(app.output).to_not match("No change in requirements detected, installing from cache")
+
+        # With no changes on redeploy, the cache should be present
+        run!(%Q{git commit --allow-empty -m next})
+        app.push!
+
+        expect(app.output).to match("No change in requirements detected, installing from cache")
+        expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies")
+        expect(app.output).to_not match("No dependencies found, preparing to install")
       end
-      `git add runtime.txt && git commit -am "setting python version"`
     end
   end
 
-  before(:each) do
+  describe "python versions" do
-    set_python_version(app.directory, python_version)
+    let(:stack) { ENV["HEROKU_TEST_STACK"] || DEFAULT_STACK }
-    init_app(app)
+    it "works with 3.7.6" do
+      version = "3.7.6"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack).deploy do |app|
+        expect(app.run('python -V')).to match(version)
+      end
+    end
+
+    it "works with 3.8.2" do
+      version = "3.8.2"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack).deploy do |app|
+        expect(app.run('python -V')).to match(version)
+      end
+    end
+
+    it "fails with a bad version" do
+      version = "3.8.2.lol"
+      before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) }
+      Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack, allow_failure: true).deploy do |app|
+        expect(app.output).to match("not available for this stack")
+      end
+    end
   end
 
-  ["3.7.6", "3.8.2"].each do |version|
+  it "getting started app has no relative paths" do
-    context "on python-#{version}" do
+    buildpacks = [
-      let(:app) { Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK) }
+      :default,
-      let(:python_version) { version }
+      "https://github.com/sharpstone/force_absolute_paths_buildpack"
-      it "🐍" do
+    ]
-        app.deploy do |app|
+    Hatchet::Runner.new("python-getting-started", buildpacks: buildpacks).deploy do |app|
-          # What should happen on first deploy
+      # Deploy works
-          expect(app.output).to           match(/Installing pip/)
-
-          # What should not happen
-          expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies")
-          expect(app.output).to_not match("No change in requirements detected, installing from cache")
-          expect(app.output).to_not match("No such file or directory")
-          expect(app.output).to_not match("cp: cannot create regular file")
-
-          # let the previous build finish
-          sleep(5)
-
-          # Redeploy with changed requirements file
-          run!(%Q{echo "" >> requirements.txt})
-          run!(%Q{echo "flask" >> requirements.txt})
-          run!(%Q{git add . ; git commit --allow-empty -m next})
-          app.push!
-
-          # Check the cache to have cleared
-          expect(app.output).to match("Requirements file has been changed, clearing cached dependencies")
-
-          # What should not happen when the requirements file is changed
-          expect(app.output).to_not match("No dependencies found, preparing to install")
-          expect(app.output).to_not match("No change in requirements detected, installing from cache")
-
-          # let the previous build finish
-          sleep(5)
-
-          run!(%Q{git commit --allow-empty -m next})
-          app.push!
-
-          # With no changes on redeploy, the cache should
-          expect(app.output).to match("No change in requirements detected, installing from cache")
-
-          # With no changes on redeploy, the cache should not
-          expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies")
-          expect(app.output).to_not match("No dependencies found, preparing to install")
-
-          expect(app.run('python -V')).to match(version)
-        end
-      end
     end
   end
 end

spec/spec_helper.rb

@@ -1,4 +1,4 @@
-ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/' + ENV['TRAVIS_REPO_SLUG'] + '.git'
+ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/heroku/heroku-buildpack-python.git'
 
 require 'rspec/core'
 require 'rspec/retry'
@@ -32,8 +32,3 @@ def run!(cmd)
   raise "Error running command #{cmd} with output: #{out}" unless $?.success?
   return out
 end
-
-def init_app(app, stack=DEFAULT_STACK)
-  app.setup!
-  app.platform_api.app.update(app.name, {"build_stack" => ENV["HEROKU_TEST_STACK"] || stack})
-end