Overhaul ansible-test sanity implementation. (#22177)

- Tests are run to completion instead of stopping on first failure.
- Test results are now parsed instead of passing through to the console.
- Test results can be saved in junit xml format.
- Test results will show up on the Shippable "Tests" result tab.
- Added an experimental --lint option for easier integration with other tools.
- Code smell tests are now usable with the --list-tests, --test and --skip-test options.
- Code split out from executor.py into sanity.py.
- Rename download-logs to download.py and add support for test and coverage results.
- Miscellaneous improvements.

test/runner/test.py

@@ -24,9 +24,7 @@ from lib.executor import (
     command_windows_integration,
     command_units,
     command_compile,
-    command_sanity,
     command_shell,
-    SANITY_TESTS,
     SUPPORTED_PYTHON_VERSIONS,
     COMPILE_PYTHON_VERSIONS,
     PosixIntegrationConfig,
@@ -42,6 +40,12 @@ from lib.executor import (
     check_startup,
 )
 
+from lib.sanity import (
+    command_sanity,
+    sanity_init,
+    sanity_get_tests,
+)
+
 from lib.target import (
     find_target_completion,
     walk_posix_integration_targets,
@@ -64,10 +68,12 @@ def main():
     try:
         git_root = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..'))
         os.chdir(git_root)
+        sanity_init()
         args = parse_args()
         config = args.config(args)
         display.verbosity = config.verbosity
         display.color = config.color
+        display.info_stderr = isinstance(config, SanityConfig) and config.lint
         check_startup()
 
         try:
@@ -267,19 +273,27 @@ def parse_args():
     sanity.add_argument('--test',
                         metavar='TEST',
                         action='append',
-                        choices=[t.name for t in SANITY_TESTS],
-                        help='tests to run')
+                        choices=[test.name for test in sanity_get_tests()],
+                        help='tests to run').completer = complete_sanity_test
 
     sanity.add_argument('--skip-test',
                         metavar='TEST',
                         action='append',
-                        choices=[t.name for t in SANITY_TESTS],
-                        help='tests to skip')
+                        choices=[test.name for test in sanity_get_tests()],
+                        help='tests to skip').completer = complete_sanity_test
 
     sanity.add_argument('--list-tests',
                         action='store_true',
                         help='list available tests')
 
+    sanity.add_argument('--lint',
+                        action='store_true',
+                        help='write lint output to stdout, everything else stderr')
+
+    sanity.add_argument('--junit',
+                        action='store_true',
+                        help='write test failures to junit xml files')
+
     sanity.add_argument('--python',
                         metavar='VERSION',
                         choices=SUPPORTED_PYTHON_VERSIONS,
@@ -540,5 +554,18 @@ def complete_network_platform(prefix, parsed_args, **_):
     return [i for i in images if i.startswith(prefix) and (not parsed_args.platform or i not in parsed_args.platform)]
 
 
+def complete_sanity_test(prefix, parsed_args, **_):
+    """
+    :type prefix: unicode
+    :type parsed_args: any
+    :rtype: list[str]
+    """
+    del parsed_args
+
+    tests = sorted(t.name for t in sanity_get_tests())
+
+    return [i for i in tests if i.startswith(prefix)]
+
+
 if __name__ == '__main__':
     main()