From a98ef91566e506a55a0aa4547c8b022a2dd8523a Mon Sep 17 00:00:00 2001
From: Ed Morley <501702+edmorley@users.noreply.github.com>
Date: Wed, 7 Oct 2020 15:10:20 +0100
Subject: [PATCH] Tests: Clean up the Python version unit tests (#1092)

* Fixes the "Installing <version>" assertions so that they don't false
  positive against the "please upgrade to <version>" output.
* Removes modification of test fixtures during tests, since it can lead
  to failures depending on test order, and confusion when debugging.
* Updates the PyPy version warning tests to use a slightly newer (but
  still not latest) PyPy version, which means that the test now passes
  on Cedar-14 and can be unskipped.
* Switches to using an empty requirements file for version tests that
  duplicate the main test, to save spending time installing dependencies
  unnecessarily.
* Switches the NLTK test to using the default buildpack Python version,
  rather than an ancient Python 3.6.
* Skips the Python 3.7/3.8 tests on Cedar-14 rather than asserting
  failure, since we know they'll never pass due to Cedar-14's libssl being
  older than required.
* Removes redundant `testSqliteInstall` test since it duplicates the
  Python version install tests.

Longer term I'll be moving many of the unit tests to Hatchet, however
this at least makes the tests more dependable in the meantime.

Closes @W-8060219@.
Closes @W-8176779@.

[skip changelog]
---
 test/fixtures/nltk/runtime.txt                |   1 -
 test/fixtures/pypy2_7_warn/runtime.txt        |   2 +-
 test/fixtures/pypy3_6_warn/runtime.txt        |   2 +-
 test/fixtures/python2_warn/requirements.txt   |   1 -
 test/fixtures/python3_4_warn/requirements.txt |   1 -
 test/fixtures/python3_5_warn/requirements.txt |   1 -
 test/fixtures/python3_6/runtime.txt           |   2 +-
 test/fixtures/python3_6_warn/requirements.txt |   1 -
 test/fixtures/python3_7/runtime.txt           |   2 +-
 test/fixtures/python3_7_warn/requirements.txt |   1 -
 test/fixtures/python3_8/runtime.txt           |   2 +-
 test/fixtures/python3_8_warn/requirements.txt |   1 -
 test/fixtures/pythonDefault/requirements.txt  |   1 -
 .../requirements.txt                          |   0
 test/run-deps                                 |   6 -
 test/run-features                             |  11 +-
 test/run-versions                             | 167 ++++++++----------
 test/utils                                    |   5 -
 18 files changed, 89 insertions(+), 118 deletions(-)
 delete mode 100644 test/fixtures/nltk/runtime.txt
 delete mode 100644 test/fixtures/pythonDefault/requirements.txt
 rename test/fixtures/{no-runtime-txt => python_version_unspecified}/requirements.txt (100%)

diff --git a/test/fixtures/nltk/runtime.txt b/test/fixtures/nltk/runtime.txt
deleted file mode 100644
index 1935e97..0000000
--- a/test/fixtures/nltk/runtime.txt
+++ /dev/null
@@ -1 +0,0 @@
-python-3.6.6
diff --git a/test/fixtures/pypy2_7_warn/runtime.txt b/test/fixtures/pypy2_7_warn/runtime.txt
index c502b68..e3d29d3 100644
--- a/test/fixtures/pypy2_7_warn/runtime.txt
+++ b/test/fixtures/pypy2_7_warn/runtime.txt
@@ -1 +1 @@
-pypy2.7-7.2.0
+pypy2.7-7.3.1
diff --git a/test/fixtures/pypy3_6_warn/runtime.txt b/test/fixtures/pypy3_6_warn/runtime.txt
index e1af38f..af949aa 100644
--- a/test/fixtures/pypy3_6_warn/runtime.txt
+++ b/test/fixtures/pypy3_6_warn/runtime.txt
@@ -1 +1 @@
-pypy3.6-7.2.0
+pypy3.6-7.3.1
diff --git a/test/fixtures/python2_warn/requirements.txt b/test/fixtures/python2_warn/requirements.txt
index 663bd1f..e69de29 100644
--- a/test/fixtures/python2_warn/requirements.txt
+++ b/test/fixtures/python2_warn/requirements.txt
@@ -1 +0,0 @@
-requests
\ No newline at end of file
diff --git a/test/fixtures/python3_4_warn/requirements.txt b/test/fixtures/python3_4_warn/requirements.txt
index 7e10602..e69de29 100644
--- a/test/fixtures/python3_4_warn/requirements.txt
+++ b/test/fixtures/python3_4_warn/requirements.txt
@@ -1 +0,0 @@
-flask
diff --git a/test/fixtures/python3_5_warn/requirements.txt b/test/fixtures/python3_5_warn/requirements.txt
index 7e10602..e69de29 100644
--- a/test/fixtures/python3_5_warn/requirements.txt
+++ b/test/fixtures/python3_5_warn/requirements.txt
@@ -1 +0,0 @@
-flask
diff --git a/test/fixtures/python3_6/runtime.txt b/test/fixtures/python3_6/runtime.txt
index 9fbd3bf..4252f10 100644
--- a/test/fixtures/python3_6/runtime.txt
+++ b/test/fixtures/python3_6/runtime.txt
@@ -1 +1 @@
-python-3.6.8
+python-3.6.12
diff --git a/test/fixtures/python3_6_warn/requirements.txt b/test/fixtures/python3_6_warn/requirements.txt
index 663bd1f..e69de29 100644
--- a/test/fixtures/python3_6_warn/requirements.txt
+++ b/test/fixtures/python3_6_warn/requirements.txt
@@ -1 +0,0 @@
-requests
\ No newline at end of file
diff --git a/test/fixtures/python3_7/runtime.txt b/test/fixtures/python3_7/runtime.txt
index a01373a..795ee72 100644
--- a/test/fixtures/python3_7/runtime.txt
+++ b/test/fixtures/python3_7/runtime.txt
@@ -1 +1 @@
-python-3.7.2
+python-3.7.9
diff --git a/test/fixtures/python3_7_warn/requirements.txt b/test/fixtures/python3_7_warn/requirements.txt
index 663bd1f..e69de29 100644
--- a/test/fixtures/python3_7_warn/requirements.txt
+++ b/test/fixtures/python3_7_warn/requirements.txt
@@ -1 +0,0 @@
-requests
\ No newline at end of file
diff --git a/test/fixtures/python3_8/runtime.txt b/test/fixtures/python3_8/runtime.txt
index 724c203..0fd6938 100644
--- a/test/fixtures/python3_8/runtime.txt
+++ b/test/fixtures/python3_8/runtime.txt
@@ -1 +1 @@
-python-3.8.2
+python-3.8.6
diff --git a/test/fixtures/python3_8_warn/requirements.txt b/test/fixtures/python3_8_warn/requirements.txt
index f229360..e69de29 100644
--- a/test/fixtures/python3_8_warn/requirements.txt
+++ b/test/fixtures/python3_8_warn/requirements.txt
@@ -1 +0,0 @@
-requests
diff --git a/test/fixtures/pythonDefault/requirements.txt b/test/fixtures/pythonDefault/requirements.txt
deleted file mode 100644
index 7e10602..0000000
--- a/test/fixtures/pythonDefault/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-flask
diff --git a/test/fixtures/no-runtime-txt/requirements.txt b/test/fixtures/python_version_unspecified/requirements.txt
similarity index 100%
rename from test/fixtures/no-runtime-txt/requirements.txt
rename to test/fixtures/python_version_unspecified/requirements.txt
diff --git a/test/run-deps b/test/run-deps
index 551d21e..8b69f2a 100755
--- a/test/run-deps
+++ b/test/run-deps
@@ -60,12 +60,6 @@ testPysqlite() {
   assertCapturedSuccess
 }
 
-testSqliteInstall() {
-  compile "pythonDefault"
-  assertNotCaptured "Sqlite3 failed to install."
-  assertCapturedSuccess
-}
-
 testCffi() {
   compile "cffi"
   assertCaptured "cffi"
diff --git a/test/run-features b/test/run-features
index 6a59a76..f67da7b 100755
--- a/test/run-features
+++ b/test/run-features
@@ -59,22 +59,23 @@ testPipenvLock() {
   assertCapturedSuccessWithStdErr
 }
 
-testPipenvVersion() {
+testPipenvPythonVersion3_6() {
   compile "pipenv-version"
-  assertCaptured $DEFAULT_PYTHON_VERSION
+  assertCaptured "Installing ${LATEST_36}"
   # Can't use `assertCapturedSuccess` since stderr contains:
   # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
   assertCapturedSuccessWithStdErr
 }
 
-testPipenvVersion2() {
+testPipenvPythonVersion2_7() {
   compile "pipenv-version2"
-  assertCaptured $LATEST_27
+  assertCaptured "Installing ${LATEST_27}"
   # Can't use `assertCapturedSuccess` since stderr contains:
   # "cp: cannot stat '/tmp/build_*/requirements.txt': No such file or directory" (W-7924941)
   assertCapturedSuccessWithStdErr
 }
-testPipenvFullVersion() {
+
+testPipenvPythonFullVersion() {
   compile "pipenv-full-version"
   assertCaptured "3.6.3"
   # Can't use `assertCapturedSuccess` since stderr contains:
diff --git a/test/run-versions b/test/run-versions
index 82b2f2b..15b6e99 100755
--- a/test/run-versions
+++ b/test/run-versions
@@ -4,42 +4,40 @@
 # shellcheck source=bin/default_pythons
 source "bin/default_pythons"
 
-testPythonDefault() {
-  updateVersion "pythonDefault" $DEFAULT_PYTHON_VERSION
-  compile "pythonDefault"
-  assertCaptured $DEFAULT_PYTHON_VERSION
+testPythonVersionUnspecified() {
+  compile "python_version_unspecified"
+  assertCaptured "Installing ${DEFAULT_PYTHON_VERSION}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
 }
 
-testPython2() {
-    updateVersion "python2" $LATEST_27
-    echo $LATEST_27 > "runtime.txt"
-    compile "python2"
-    assertCaptured $LATEST_27
-    assertCaptured "python-2-7-eol-faq";
-    assertNotCaptured "security update"
-    assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+testPython2_7() {
+  compile "python2"
+  assertCaptured "Installing ${LATEST_27}"
+  assertCaptured "python-2-7-eol-faq";
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }
 
-testPython2_warn() {
-    compile "python2_warn"
-    assertCaptured "python-2.7.15"
-    assertCaptured "python-2-7-eol-faq";
-    assertCaptured "Only the latest version"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+testPython2_7_warn() {
+  compile "python2_warn"
+  assertCaptured "Installing python-2.7.15"
+  assertCaptured "python-2-7-eol-faq";
+  assertCaptured "Only the latest version"
+  assertCaptured "${LATEST_27}"
+  assertCapturedSuccess
 }
 
 testPython3_4() {
   compile "python3_4"
-  assertCaptured $LATEST_34
+  assertCaptured "Installing ${LATEST_34}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 19.1.1, setuptools 43.0.0 and wheel 0.33.6"
+  assertCaptured "Installing SQLite3"
   # Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
   # and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
   assertCapturedSuccessWithStdErr
@@ -47,8 +45,9 @@ testPython3_4() {
 
 testPython3_4_warn() {
   compile "python3_4_warn"
-  assertCaptured "python-3.4.9"
+  assertCaptured "Installing python-3.4.9"
   assertCaptured "security update!"
+  assertCaptured "${LATEST_34}"
   # Can't use `assertCapturedSuccess` since Pip outputs a Python 3.4 EOL warning to stderr,
   # and the newest Pip that works on Python 3.4 doesn't support `PIP_NO_PYTHON_VERSION_WARNING`.
   assertCapturedSuccessWithStdErr
@@ -56,7 +55,7 @@ testPython3_4_warn() {
 
 testPython3_5() {
   compile "python3_5"
-  assertCaptured $LATEST_35
+  assertCaptured "Installing ${LATEST_35}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
@@ -65,15 +64,15 @@ testPython3_5() {
 
 testPython3_5_warn() {
   compile "python3_5_warn"
-  assertCaptured "python-3.5.6"
+  assertCaptured "Installing python-3.5.6"
   assertCaptured "security update!"
+  assertCaptured "${LATEST_35}"
   assertCapturedSuccess
 }
 
 testPython3_6() {
-  updateVersion "python3_6" $LATEST_36
   compile "python3_6"
-  assertCaptured $LATEST_36
+  assertCaptured "Installing ${LATEST_36}"
   assertNotCaptured "security update"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
@@ -82,62 +81,60 @@ testPython3_6() {
 
 testPython3_6_warn() {
   compile "python3_6_warn"
-  assertCaptured "python-3.6.7"
+  assertCaptured "Installing python-3.6.7"
   assertCaptured "security update!"
-  assertCaptured "Installing SQLite3"
+  assertCaptured "${LATEST_36}"
   assertCapturedSuccess
 }
 
 testPython3_7() {
-  updateVersion "python3_7" $LATEST_37
-  compile "python3_7"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertNotCaptured "security update"
-    assertCaptured $LATEST_37
-    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+  # Python 3.7+ requires newer libssl than is present on Cedar-14.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
   fi
+  compile "python3_7"
+  assertCaptured "Installing ${LATEST_37}"
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }
 
 testPython3_7_warn() {
-  compile "python3_7_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "python-3.7.1"
-    assertCaptured "security update!"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+  # Python 3.7+ requires newer libssl than is present on Cedar-14.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
   fi
+  compile "python3_7_warn"
+  assertCaptured "Installing python-3.7.1"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_37}"
+  assertCapturedSuccess
 }
 
 testPython3_8() {
-  updateVersion "python3_8" $LATEST_38
-  compile "python3_8"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertNotCaptured "security update"
-    assertCaptured $LATEST_38
-    assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+  # Python 3.7+ requires newer libssl than is present on Cedar-14.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
   fi
+  compile "python3_8"
+  assertCaptured "Installing ${LATEST_38}"
+  assertNotCaptured "security update"
+  assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
+  assertCaptured "Installing SQLite3"
+  assertCapturedSuccess
 }
 
 testPython3_8_warn() {
-  compile "python3_8_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "python-3.8.0"
-    assertCaptured "security update!"
-    assertCaptured "Installing SQLite3"
-    assertCapturedSuccess
+  # Python 3.7+ requires newer libssl than is present on Cedar-14.
+  if [[ "${STACK}" = "cedar-14" ]]; then
+    return
   fi
+  compile "python3_8_warn"
+  assertCaptured "Installing python-3.8.0"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_38}"
+  assertCapturedSuccess
 }
 
 testPython3_9() {
@@ -146,8 +143,8 @@ testPython3_9() {
     return
   fi
   compile "python3_9"
+  assertCaptured "Installing ${LATEST_39}"
   assertNotCaptured "security update"
-  assertCaptured $LATEST_39
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCaptured "Installing SQLite3"
   assertCapturedSuccess
@@ -163,8 +160,9 @@ testPython3_9_warn() {
     return
   fi
   compile "python3_9_warn"
-  assertCaptured "python-3.9.0"
+  assertCaptured "Installing python-3.9.0"
   assertCaptured "security update!"
+  assertCaptured "${LATEST_39}"
   assertCapturedSuccess
 }
 
@@ -177,44 +175,34 @@ testPythonVersionInvalid() {
 
 testPypy3_6() {
   compile "pypy3_6"
-  assertCaptured "Installing pypy"
+  assertCaptured "Installing ${LATEST_PYPY_36}"
   assertNotCaptured "security update"
-  assertCaptured "$LATEST_PYPY_36"
   assertCaptured "Installing pip 20.1.1, setuptools 47.1.1 and wheel 0.34.2"
   assertCapturedSuccess
 }
 
 testPypy3_6_warn() {
   compile "pypy3_6_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "Installing pypy"
-    assertCaptured "security update!"
-    assertCaptured "$LATEST_PYPY_36"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing pypy3.6-7.3.1"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_PYPY_36}"
+  assertCapturedSuccess
 }
 
 testPypy2_7() {
   compile "pypy2_7"
-  assertCaptured "Installing pypy"
+  assertCaptured "Installing ${LATEST_PYPY_27}"
   assertNotCaptured "security update"
-  assertCaptured "$LATEST_PYPY_27"
   assertCaptured "Installing pip 20.1.1, setuptools 44.1.1 and wheel 0.34.2"
   assertCapturedSuccess
 }
 
 testPypy2_7_warn() {
   compile "pypy2_7_warn"
-  if [[ $STACK = "cedar-14" ]]; then
-    assertCapturedError
-  else
-    assertCaptured "Installing pypy"
-    assertCaptured "security update!"
-    assertCaptured "$LATEST_PYPY_27"
-    assertCapturedSuccess
-  fi
+  assertCaptured "Installing pypy2.7-7.3.1"
+  assertCaptured "security update!"
+  assertCaptured "${LATEST_PYPY_27}"
+  assertCapturedSuccess
 }
 
 testStickyPythonVersion() {
@@ -222,8 +210,9 @@ testStickyPythonVersion() {
   compile "python3_6_warn" "$cache_dir"
   assertCaptured "Installing python-3.6.7"
   assertCapturedSuccess
-  compile "no-runtime-txt" "$cache_dir"
-  assertCaptured "Installing python-3.6.7"
+  compile "python_version_unspecified" "$cache_dir"
+  assertNotCaptured "Installing python"
+  assertCaptured "security update!"
   assertCapturedSuccess
   # Whilst this file seems like an implementation detail (so something that should
   # not be tested), we must guarantee the filename remains consistent for backwards
diff --git a/test/utils b/test/utils
index 297a664..af7db6f 100644
--- a/test/utils
+++ b/test/utils
@@ -56,11 +56,6 @@ resetCapture()
   unset rtrn # deprecated
 }
 
-updateVersion()
-{
-  echo "$2" > "test/fixtures/${1}/runtime.txt"
-}
-
 assertCapturedEquals()
 {
   assertEquals "$@" "$(cat ${STD_OUT})"