Increase python version coverage for tests. (#24762)

* Improve ansible-test inventory handling.
* Fix python 3 re-raise of exception from thread.
* Fix python 3 encoding for windows-integration.
* Run network tests on multiple python versions.
* Run windows tests on multiple python versions.
* Support Shippable delegation using --tox.
* Skip vyos_command on python 3 tests until fixed.
* Add python 3 filtering to local and tox.
* Fix tests to support back to back runs.
* Temporarily test networking with python 2.7 only.

Running the tests back to back causes intermittent test failures
which need to be addressed before we can test multiple versions
in a single test run.

test/utils/shippable/network.sh

@@ -12,22 +12,38 @@ fi
 
 target="network/ci/"
 
+# python versions to test in order
+# all versions run full tests
+python_versions=(
+    2.7
+)
+
 if [ -s /tmp/network.txt ]; then
     echo "Detected changes requiring integration tests specific to networking:"
     cat /tmp/network.txt
 
     echo "Running network integration tests for multiple platforms concurrently."
 
-    # shellcheck disable=SC2086
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
-        --platform vyos/1.1.0 \
-        --platform ios/csr1000v \
-
+    platforms=(
+        --platform vyos/1.1.0
+        --platform ios/csr1000v
+    )
 else
     echo "No changes requiring integration tests specific to networking were detected."
     echo "Running network integration tests for a single platform only."
 
-    # shellcheck disable=SC2086
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+    platforms=(
         --platform vyos/1.1.0
+    )
 fi
+
+retry.py pip install tox --disable-pip-version-check
+
+for version in "${python_versions[@]}"; do
+    # clean up between test runs until we switch from --tox to --docker
+    rm -rf ~/.ansible/{cp,pc,tmp}/
+
+    # shellcheck disable=SC2086
+    ansible-test network-integration --color -v --retry-on-error "${target}" --tox --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+        "${platforms[@]}"
+done

test/utils/shippable/windows.sh

@@ -7,6 +7,15 @@ IFS='/:' read -ra args <<< "${TEST}"
 
 job="${args[1]}"
 
+# python versions to test in order
+# python 2.7 runs full tests while other versions run minimal tests
+python_versions=(
+    2.6
+    3.5
+    3.6
+    2.7
+)
+
 # shellcheck disable=SC2086
 ansible-test windows-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
 
@@ -23,20 +32,40 @@ if [ -s /tmp/windows.txt ]; then
 
     target="windows/ci/"
 
-    # shellcheck disable=SC2086
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
-        --windows 2008-SP2 \
-        --windows 2008-R2_SP1 \
-        --windows 2012-RTM \
-        --windows 2012-R2_RTM \
-
+    platforms=(
+        --windows 2008-SP2
+        --windows 2008-R2_SP1
+        --windows 2012-RTM
+        --windows 2012-R2_RTM
+    )
 else
     echo "No changes requiring integration tests specific to Windows were detected."
     echo "Running Windows integration tests for a single version only."
 
     target="windows/ci/group${job}/"
 
-    # shellcheck disable=SC2086
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+    platforms=(
         --windows 2012-R2_RTM
+    )
 fi
+
+retry.py pip install tox --disable-pip-version-check
+
+for version in "${python_versions[@]}"; do
+    # clean up between test runs until we switch from --tox to --docker
+    rm -rf ~/.ansible/{cp,pc,tmp}/
+
+    if [ "${job}" == "1" ] || [ "${version}" == "2.7" ]; then
+        if [ "${version}" == "2.7" ]; then
+            # full tests for python 2.7
+            ci="${target}"
+        else
+            # minimal tests for other python versions
+            ci="windows/ci/minimal/"
+        fi
+
+        # shellcheck disable=SC2086
+        ansible-test windows-integration --color -v --retry-on-error "${ci}" --tox --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+            "${platforms[@]}"
+    fi
+done