reorganize downstream tests to avoid dependency squashing (#4418)

* reorganize downstream tests

* fix run.sh syntax

* add instructions for adding more downstream tests

* rework downstream CI test guide into rst readme

* remove unnecessary example test handler

* all test handlers should "exit 1" if an unexpected argument is received

10 files changed, 154 insertions, 60 deletions

diff --git a/.travis/downstream.d/README.rst b/.travis/downstream.d/README.rst
new file mode 100644
index 00000000..1553448c
--- /dev/null
+++ b/.travis/downstream.d/README.rst
@@ -0,0 +1,13 @@
+To add downstream tests to be run in CI:
+
+1. Create a test handler for the downstream consumer that you want to test.
+
+   * The test handler should be a single file in the ``.travis/downstream.d/`` directory.
+   * The file name should be ``{downstream name}.sh`` where ``{downstream name}``
+     is the name that you wish to use to identify the consumer.
+   * The test handler should accept a single argument that can be either ``install`` or ``run``.
+     These should be used to separate installation of the downstream consumer and
+     any dependencies from the actual running of the tests.
+
+2. Add an entry to the test matrix in ``.travis.yml`` that sets the ``DOWNSTREAM``
+   environment variable to the downstream name that you selected.
diff --git a/.travis/downstream.d/aws-encryption-sdk.sh b/.travis/downstream.d/aws-encryption-sdk.sh
new file mode 100755
index 00000000..e478cd8a
--- /dev/null
+++ b/.travis/downstream.d/aws-encryption-sdk.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+case "${1}" in
+    install)
+        git clone --depth=1 https://github.com/awslabs/aws-encryption-sdk-python
+        cd aws-encryption-sdk-python
+        pip install -r test/requirements.txt
+        pip install -e .
+        ;;
+    run)
+        cd aws-encryption-sdk-python
+        pytest -m local test/
+        ;;
+    *)
+        exit 1
+        ;;
+esac
diff --git a/.travis/downstream.d/certbot-josepy.sh b/.travis/downstream.d/certbot-josepy.sh
new file mode 100755
index 00000000..47de2e29
--- /dev/null
+++ b/.travis/downstream.d/certbot-josepy.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+case "${1}" in
+    install)
+        git clone --depth=1 https://github.com/certbot/josepy
+        cd josepy
+        pip install -e ".[tests]"
+        ;;
+    run)
+        cd josepy
+        pytest src
+        ;;
+    *)
+        exit 1
+        ;;
+esac
diff --git a/.travis/downstream.d/certbot.sh b/.travis/downstream.d/certbot.sh
new file mode 100755
index 00000000..f255600f
--- /dev/null
+++ b/.travis/downstream.d/certbot.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+case "${1}" in
+    install)
+        git clone --depth=1 https://github.com/certbot/certbot
+        cd certbot
+        pip install pytest pytest-mock mock
+        pip install -e acme
+        pip install -e .
+        ;;
+    run)
+        cd certbot
+        pytest certbot/tests
+        pytest acme
+        ;;
+    *)
+        exit 1
+        ;;
+esac
diff --git a/.travis/downstream.d/dynamodb-encryption-sdk.sh b/.travis/downstream.d/dynamodb-encryption-sdk.sh
new file mode 100755
index 00000000..a7d3885b
--- /dev/null
+++ b/.travis/downstream.d/dynamodb-encryption-sdk.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+case "${1}" in
+    install)
+        git clone --depth=1 https://github.com/awslabs/aws-dynamodb-encryption-python
+        cd aws-dynamodb-encryption-python
+        pip install -r test/requirements.txt
+        pip install -e .
+        ;;
+    run)
+        cd aws-dynamodb-encryption-python
+        pytest -m "local and not slow and not veryslow and not nope"
+        ;;
+    *)
+        exit 1
+        ;;
+esac
diff --git a/.travis/downstream.d/paramiko.sh b/.travis/downstream.d/paramiko.sh
new file mode 100755
index 00000000..ccba686d
--- /dev/null
+++ b/.travis/downstream.d/paramiko.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+case "${1}" in
+    install)
+        git clone --depth=1 https://github.com/paramiko/paramiko
+        cd paramiko
+        pip install -e .
+        pip install -r dev-requirements.txt
+        ;;
+    run)
+        cd paramiko
+        inv test
+        ;;
+    *)
+        exit 1
+        ;;
+esac
diff --git a/.travis/downstream.d/pyopenssl.sh b/.travis/downstream.d/pyopenssl.sh
new file mode 100755
index 00000000..b95bb280
--- /dev/null
+++ b/.travis/downstream.d/pyopenssl.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+case "${1}" in
+    install)
+        git clone --depth=1 https://github.com/pyca/pyopenssl
+        cd pyopenssl
+        pip install -e ".[test]"
+        ;;
+    run)
+        cd pyopenssl
+        pytest tests
+        ;;
+    *)
+        exit 1
+        ;;
+esac
diff --git a/.travis/downstream.d/twisted.sh b/.travis/downstream.d/twisted.sh
new file mode 100755
index 00000000..cb25027a
--- /dev/null
+++ b/.travis/downstream.d/twisted.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+case "${1}" in
+    install)
+        git clone --depth=1 https://github.com/twisted/twisted
+        cd twisted
+        pip install -e .[tls,conch,http2]
+        ;;
+    run)
+        cd twisted
+        python -m twisted.trial src/twisted
+        ;;
+    *)
+        exit 1
+        ;;
+esac
diff --git a/.travis/downstream.d/urllib3.sh b/.travis/downstream.d/urllib3.sh
new file mode 100755
index 00000000..8442d1e7
--- /dev/null
+++ b/.travis/downstream.d/urllib3.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+case "${1}" in
+    install)
+        git clone --depth 1 https://github.com/shazow/urllib3
+        cd urllib3
+        pip install -r ./dev-requirements.txt
+        pip install -e ".[socks]"
+        ;;
+    run)
+        cd urllib3
+        pytest test
+        ;;
+    *)
+        exit 1
+        ;;
+esac
diff --git a/.travis/run.sh b/.travis/run.sh
index 572b9a4d..ca0ddac6 100755
--- a/.travis/run.sh
+++ b/.travis/run.sh
@@ -26,65 +26,11 @@ source ~/.venv/bin/activate
 if [ -n "${TOXENV}" ]; then
     tox -- --wycheproof-root=$HOME/wycheproof
 else
+    downstream_script="${TRAVIS_BUILD_DIR}/.travis/downstream.d/${DOWNSTREAM}.sh"
+    if [ ! -x $downstream_script ]; then
+        exit 1
+    fi
+    $downstream_script install
     pip install .
-    case "${DOWNSTREAM}" in
-        pyopenssl)
-            git clone --depth=1 https://github.com/pyca/pyopenssl
-            cd pyopenssl
-            pip install -e ".[test]"
-            pytest tests
-            ;;
-        twisted)
-            git clone --depth=1 https://github.com/twisted/twisted
-            cd twisted
-            pip install -e .[tls,conch,http2]
-            python -m twisted.trial src/twisted
-            ;;
-        paramiko)
-            git clone --depth=1 https://github.com/paramiko/paramiko
-            cd paramiko
-            pip install -e .
-            pip install -r dev-requirements.txt
-            inv test
-            ;;
-        aws-encryption-sdk)
-            git clone --depth=1 https://github.com/awslabs/aws-encryption-sdk-python
-            cd aws-encryption-sdk-python
-            pip install -r test/requirements.txt
-            pip install -e .
-            pytest -m local test/
-            ;;
-        dynamodb-encryption-sdk)
-            git clone --depth=1 https://github.com/awslabs/aws-dynamodb-encryption-python
-            cd aws-dynamodb-encryption-python
-            pip install -r test/requirements.txt
-            pip install -e .
-            pytest -m "local and not slow and not veryslow and not nope"
-            ;;
-        certbot)
-            git clone --depth=1 https://github.com/certbot/certbot
-            cd certbot
-            pip install pytest pytest-mock mock
-            pip install -e acme
-            pip install -e .
-            pytest certbot/tests
-            pytest acme
-            ;;
-        certbot-josepy)
-            git clone --depth=1 https://github.com/certbot/josepy
-            cd josepy
-            pip install -e ".[tests]"
-            pytest src
-            ;;
-        urllib3)
-            git clone --depth 1 https://github.com/shazow/urllib3
-            cd urllib3
-            pip install -r ./dev-requirements.txt
-            pip install -e ".[socks]"
-            pytest test
-            ;;
-        *)
-            exit 1
-            ;;
-    esac
+    $downstream_script run
 fi