Increase python version coverage for tests. (#24762)

* Improve ansible-test inventory handling.
* Fix python 3 re-raise of exception from thread.
* Fix python 3 encoding for windows-integration.
* Run network tests on multiple python versions.
* Run windows tests on multiple python versions.
* Support Shippable delegation using --tox.
* Skip vyos_command on python 3 tests until fixed.
* Add python 3 filtering to local and tox.
* Fix tests to support back to back runs.
* Temporarily test networking with python 2.7 only.

Running the tests back to back causes intermittent test failures
which need to be addressed before we can test multiple versions
in a single test run.

test/runner/lib/core_ci.py

@@ -265,8 +265,8 @@ class AnsibleCoreCI(object):
         display.info('Initializing new %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1)
 
         if self.platform == 'windows':
-            with open('examples/scripts/ConfigureRemotingForAnsible.ps1', 'r') as winrm_config_fd:
-                winrm_config = winrm_config_fd.read()
+            with open('examples/scripts/ConfigureRemotingForAnsible.ps1', 'rb') as winrm_config_fd:
+                winrm_config = winrm_config_fd.read().decode('utf-8')
         else:
             winrm_config = None
 

test/runner/lib/delegation.py

@@ -35,6 +35,8 @@ from lib.util import (
     ApplicationError,
     EnvironmentConfig,
     run_command,
+    common_environment,
+    pass_vars,
 )
 
 from lib.docker_util import (
@@ -129,7 +131,18 @@ def delegate_tox(args, exclude, require):
             if args.coverage and not args.coverage_label:
                 cmd += ['--coverage-label', 'tox-%s' % version]
 
-        run_command(args, tox + cmd)
+        env = common_environment()
+
+        # temporary solution to permit ansible-test delegated to tox to provision remote resources
+        optional = (
+            'SHIPPABLE',
+            'SHIPPABLE_BUILD_ID',
+            'SHIPPABLE_JOB_NUMBER',
+        )
+
+        env.update(pass_vars(required=[], optional=optional))
+
+        run_command(args, tox + cmd, env=env)
 
 
 def delegate_docker(args, exclude, require):

test/runner/lib/executor.py

@@ -245,6 +245,11 @@ def command_network_integration(args):
     """
     :type args: NetworkIntegrationConfig
     """
+    filename = 'test/integration/inventory.networking'
+
+    if not args.explain and not args.platform and not os.path.isfile(filename):
+        raise ApplicationError('Use the --platform option or provide an inventory file (see %s.template).' % filename)
+
     internal_targets = command_integration_filter(args, walk_network_integration_targets())
     platform_targets = set(a for t in internal_targets for a in t.aliases if a.startswith('network/'))
 
@@ -276,8 +281,6 @@ def command_network_integration(args):
         remotes = [instance.wait_for_result() for instance in instances]
         inventory = network_inventory(remotes)
 
-        filename = 'test/integration/inventory.networking'
-
         display.info('>>> Inventory: %s\n%s' % (filename, inventory.strip()), verbosity=3)
 
         if not args.explain:
@@ -349,6 +352,11 @@ def command_windows_integration(args):
     """
     :type args: WindowsIntegrationConfig
     """
+    filename = 'test/integration/inventory.winrm'
+
+    if not args.explain and not args.windows and not os.path.isfile(filename):
+        raise ApplicationError('Use the --windows option or provide an inventory file (see %s.template).' % filename)
+
     internal_targets = command_integration_filter(args, walk_windows_integration_targets())
 
     if args.windows:
@@ -368,8 +376,6 @@ def command_windows_integration(args):
         remotes = [instance.wait_for_result() for instance in instances]
         inventory = windows_inventory(remotes)
 
-        filename = 'test/integration/inventory.winrm'
-
         display.info('>>> Inventory: %s\n%s' % (filename, inventory.strip()), verbosity=3)
 
         if not args.explain:
@@ -1111,6 +1117,14 @@ def get_integration_local_filter(args, targets):
             display.warning('Excluding tests marked "%s" which require --allow-destructive to run locally: %s'
                             % (skip.rstrip('/'), ', '.join(skipped)))
 
+    if args.python_version.startswith('3'):
+        skip = 'skip/python3/'
+        skipped = [target.name for target in targets if skip in target.aliases]
+        if skipped:
+            exclude.append(skip)
+            display.warning('Excluding tests marked "%s" which are not yet supported on python 3: %s'
+                            % (skip.rstrip('/'), ', '.join(skipped)))
+
     return exclude
 
 

test/runner/lib/thread.py

@@ -43,7 +43,7 @@ class WrappedThread(threading.Thread):
         result, exception = self._result.get()
         if exception:
             if sys.version_info[0] > 2:
-                raise exception[0](exception[1]).with_traceback(exception[2])
+                raise exception[1].with_traceback(exception[2])
             # noinspection PyRedundantParentheses
             exec('raise exception[0], exception[1], exception[2]')  # pylint: disable=locally-disabled, exec-used
         return result

test/runner/lib/util.py

@@ -208,7 +208,7 @@ def common_environment():
     return env
 
 
-def pass_vars(required=None, optional=None):
+def pass_vars(required, optional):
     """
     :type required: collections.Iterable[str]
     :type optional: collections.Iterable[str]