diff --git a/test/fixtures/python2_warn/requirements.txt b/test/fixtures/python2_warn/requirements.txt
new file mode 100644
index 0000000..663bd1f
--- /dev/null
+++ b/test/fixtures/python2_warn/requirements.txt
@@ -0,0 +1 @@
+requests
\ No newline at end of file
diff --git a/test/fixtures/python2_warn/runtime.txt b/test/fixtures/python2_warn/runtime.txt
new file mode 100644
index 0000000..5385682
--- /dev/null
+++ b/test/fixtures/python2_warn/runtime.txt
@@ -0,0 +1 @@
+python-2.7.14
diff --git a/test/fixtures/python3_4_warn/requirements.txt b/test/fixtures/python3_4_warn/requirements.txt
new file mode 100644
index 0000000..7e10602
--- /dev/null
+++ b/test/fixtures/python3_4_warn/requirements.txt
@@ -0,0 +1 @@
+flask
diff --git a/test/fixtures/python3_4_warn/runtime.txt b/test/fixtures/python3_4_warn/runtime.txt
new file mode 100644
index 0000000..fe5dcf7
--- /dev/null
+++ b/test/fixtures/python3_4_warn/runtime.txt
@@ -0,0 +1 @@
+python-3.4.0
diff --git a/test/fixtures/python3_5_warn/requirements.txt b/test/fixtures/python3_5_warn/requirements.txt
new file mode 100644
index 0000000..7e10602
--- /dev/null
+++ b/test/fixtures/python3_5_warn/requirements.txt
@@ -0,0 +1 @@
+flask
diff --git a/test/fixtures/python3_5_warn/runtime.txt b/test/fixtures/python3_5_warn/runtime.txt
new file mode 100644
index 0000000..5486d7a
--- /dev/null
+++ b/test/fixtures/python3_5_warn/runtime.txt
@@ -0,0 +1 @@
+python-3.5.3
diff --git a/test/fixtures/python3_6_warn/requirements.txt b/test/fixtures/python3_6_warn/requirements.txt
new file mode 100644
index 0000000..663bd1f
--- /dev/null
+++ b/test/fixtures/python3_6_warn/requirements.txt
@@ -0,0 +1 @@
+requests
\ No newline at end of file
diff --git a/test/fixtures/python3_6_warn/runtime.txt b/test/fixtures/python3_6_warn/runtime.txt
new file mode 100644
index 0000000..486fcce
--- /dev/null
+++ b/test/fixtures/python3_6_warn/runtime.txt
@@ -0,0 +1 @@
+python-3.6.5
diff --git a/test/run b/test/run
index 67d4113..484f241 100755
--- a/test/run
+++ b/test/run
@@ -87,19 +87,48 @@ testPylibmc() {
   assertCapturedSuccess
 }
 
+testPython2_warn() {
+    compile "python2_warn"
+    assertCaptured "python-2.7.14"
+    assertCaptured "security update!"
+    assertCapturedSuccess
+}
+
 testPython2() {
     compile "python2"
     assertCaptured "python-2.7.15"
     assertCapturedSuccess
 }
 
-# This will succeed on Cedar-14 and fail on 16 and 18
+# This fail
+testPython3_4_warn() {
+  compile "python3_4_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCaptured "python-3.4.0"
+    assertCapturedSuccess
+  else
+    assertCapturedError
+  fi
+}
+
+# This fail
 testPython3_4() {
   compile "python3_4"
   assertCaptured "python-3.4.9"
   assertCapturedError
 }
 
+# This fail
+testPython3_5_warn() {
+  compile "python3_5_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCaptured "python-3.5.3"
+    assertCapturedSuccess
+  else
+    assertCapturedError
+  fi
+}
+
 # This will fail
 testPython3_5() {
   compile "python3_5"
@@ -107,6 +136,14 @@ testPython3_5() {
   assertCapturedError
 }
 
+# This will warn
+testPython3_6_warn() {
+  compile "python3_6_warn"
+  assertCaptured "python-3.6.5"
+  assertCaptured "security update!"
+  assertCapturedSuccess
+}
+
 testPython3_6() {
   compile "python3_6"
   assertCaptured "python-3.6.6"
@@ -122,8 +159,12 @@ testPython3_6_7() {
 
 testPython3_7() {
   compile "python3_7"
-  assertCaptured "python-3.7.0"
-  assertCapturedSuccess
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "python-3.7.0"
+    assertCapturedSuccess
+  fi
 }
 
 # This will fail