Update ansible-test sanity command. (#31958)

* Use correct pip version in ansible-test.
* Add git fallback for validate-modules.
* Run sanity tests in a docker container.
* Use correct python version for sanity tests.
* Pin docker completion images and add default.
* Split pylint execution into multiple contexts.
* Only test .py files in use-argspec-type-path test.
* Accept identical python interpeter name or binary.
* Switch cloud tests to default container.
* Remove unused extras from pip install.
* Filter out empty pip commands.
* Don't force running of pip list.
* Support delegation for windows and network tests.
* Fix ansible-test python version usage.
* Fix ansible-test python version skipping.
* Use absolute path for log in ansible-test.
* Run vyos_command test on python 3.
* Fix windows/network instance persistence.
* Add `test/cache` dir to classification.
* Enable more python versions for network tests.
* Fix cs_router test.

test/runner/lib/sanity/__init__.py

@@ -86,7 +86,7 @@ def command_sanity(args):
             versions = (None,)
 
         for version in versions:
-            if args.python and version and version != args.python:
+            if args.python and version and version != args.python_version:
                 continue
 
             display.info('Sanity check using %s%s' % (test.name, ' with Python %s' % version if version else ''))

test/runner/lib/sanity/ansible_doc.py

@@ -11,16 +11,13 @@ from lib.sanity import (
 from lib.util import (
     SubprocessError,
     display,
+    intercept_command,
 )
 
 from lib.ansible_util import (
     ansible_environment,
 )
 
-from lib.executor import (
-    intercept_command,
-)
-
 from lib.config import (
     SanityConfig,
 )

test/runner/lib/sanity/import.py

@@ -15,6 +15,7 @@ from lib.sanity import (
 from lib.util import (
     SubprocessError,
     run_command,
+    intercept_command,
     remove_tree,
 )
 
@@ -23,7 +24,6 @@ from lib.ansible_util import (
 )
 
 from lib.executor import (
-    intercept_command,
     generate_pip_install,
 )
 
@@ -83,7 +83,7 @@ class ImportTest(SanityMultipleVersion):
 
         # make sure coverage is available in the virtual environment if needed
         if args.coverage:
-            run_command(args, generate_pip_install('sanity.import', packages=['coverage']), env=env)
+            run_command(args, generate_pip_install('pip', 'sanity.import', packages=['coverage']), env=env)
             run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'pip'], env=env)
 
         cmd = ['importer.py'] + paths

test/runner/lib/sanity/pep8.py

@@ -15,6 +15,7 @@ from lib.util import (
     SubprocessError,
     display,
     run_command,
+    find_executable,
 )
 
 from lib.config import (
@@ -55,7 +56,8 @@ class Pep8Test(SanitySingleVersion):
         paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths_set)
 
         cmd = [
-            'pycodestyle',
+            'python%s' % args.python_version,
+            find_executable('pycodestyle'),
             '--max-line-length', '160',
             '--config', '/dev/null',
             '--ignore', ','.join(sorted(current_ignore)),

test/runner/lib/sanity/pylint.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import, print_function
 
 import json
 import os
+import datetime
 
 from lib.sanity import (
     SanitySingleVersion,
@@ -16,6 +17,7 @@ from lib.util import (
     SubprocessError,
     run_command,
     display,
+    find_executable,
 )
 
 from lib.ansible_util import (
@@ -52,51 +54,54 @@ class PylintTest(SanitySingleVersion):
         with open(PYLINT_SKIP_PATH, 'r') as skip_fd:
             skip_paths = skip_fd.read().splitlines()
 
-        with open('test/sanity/pylint/disable.txt', 'r') as disable_fd:
-            disable = set(c for c in disable_fd.read().splitlines() if not c.strip().startswith('#'))
-
-        with open('test/sanity/pylint/enable.txt', 'r') as enable_fd:
-            enable = set(c for c in enable_fd.read().splitlines() if not c.strip().startswith('#'))
-
         skip_paths_set = set(skip_paths)
 
         paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths_set)
 
-        cmd = [
-            'pylint',
-            '--jobs', '0',
-            '--reports', 'n',
-            '--max-line-length', '160',
-            '--rcfile', '/dev/null',
-            '--ignored-modules', '_MovedItems',
-            '--output-format', 'json',
-            '--disable', ','.join(sorted(disable)),
-            '--enable', ','.join(sorted(enable)),
-        ] + paths
+        contexts = {}
+        remaining_paths = set(paths)
 
-        env = ansible_environment(args)
+        def add_context(available_paths, context_name, context_filter):
+            """
+            :type available_paths: set[str]
+            :type context_name: str
+            :type context_filter: (str) -> bool
+            """
+            filtered_paths = set(p for p in available_paths if context_filter(p))
+            contexts[context_name] = sorted(filtered_paths)
+            available_paths -= filtered_paths
 
-        if paths:
-            try:
-                stdout, stderr = run_command(args, cmd, env=env, capture=True)
-                status = 0
-            except SubprocessError as ex:
-                stdout = ex.stdout
-                stderr = ex.stderr
-                status = ex.status
+        add_context(remaining_paths, 'ansible-test', lambda p: p.startswith('test/runner/'))
+        add_context(remaining_paths, 'units', lambda p: p.startswith('test/units/'))
+        add_context(remaining_paths, 'test', lambda p: p.startswith('test/'))
+        add_context(remaining_paths, 'hacking', lambda p: p.startswith('hacking/'))
+        add_context(remaining_paths, 'modules', lambda p: p.startswith('lib/ansible/modules/'))
+        add_context(remaining_paths, 'module_utils', lambda p: p.startswith('lib/ansible/module_utils/'))
+        add_context(remaining_paths, 'ansible', lambda p: True)
 
-            if stderr or status >= 32:
-                raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)
-        else:
-            stdout = None
+        messages = []
+        context_times = []
 
-        if args.explain:
-            return SanitySuccess(self.name)
+        test_start = datetime.datetime.utcnow()
 
-        if stdout:
-            messages = json.loads(stdout)
-        else:
-            messages = []
+        for context in sorted(contexts):
+            context_paths = contexts[context]
+
+            if not context_paths:
+                continue
+
+            context_start = datetime.datetime.utcnow()
+            messages += self.pylint(args, context, context_paths)
+            context_end = datetime.datetime.utcnow()
+
+            context_times.append('%s: %d (%s)' % (context, len(context_paths), context_end - context_start))
+
+        test_end = datetime.datetime.utcnow()
+
+        for context_time in context_times:
+            display.info(context_time, verbosity=4)
+
+        display.info('total: %d (%s)' % (len(paths), test_end - test_start), verbosity=4)
 
         errors = [SanityMessage(
             message=m['message'].replace('\n', ' '),
@@ -127,3 +132,48 @@ class PylintTest(SanitySingleVersion):
             return SanityFailure(self.name, messages=errors)
 
         return SanitySuccess(self.name)
+
+    def pylint(self, args, context, paths):
+        """
+        :type args: SanityConfig
+        :param context: str
+        :param paths: list[str]
+        :return: list[dict[str, str]]
+        """
+        rcfile = 'test/sanity/pylint/config/%s' % context
+
+        if not os.path.exists(rcfile):
+            rcfile = 'test/sanity/pylint/config/default'
+
+        cmd = [
+            'python%s' % args.python_version,
+            find_executable('pylint'),
+            '--jobs', '0',
+            '--reports', 'n',
+            '--max-line-length', '160',
+            '--rcfile', rcfile,
+            '--output-format', 'json',
+        ] + paths
+
+        env = ansible_environment(args)
+
+        if paths:
+            try:
+                stdout, stderr = run_command(args, cmd, env=env, capture=True)
+                status = 0
+            except SubprocessError as ex:
+                stdout = ex.stdout
+                stderr = ex.stderr
+                status = ex.status
+
+            if stderr or status >= 32:
+                raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)
+        else:
+            stdout = None
+
+        if not args.explain and stdout:
+            messages = json.loads(stdout)
+        else:
+            messages = []
+
+        return messages

test/runner/lib/sanity/rstcheck.py

@@ -15,6 +15,7 @@ from lib.util import (
     SubprocessError,
     run_command,
     parse_to_dict,
+    find_executable,
 )
 
 from lib.config import (
@@ -39,7 +40,8 @@ class RstcheckTest(SanitySingleVersion):
             return SanitySkipped(self.name)
 
         cmd = [
-            'rstcheck',
+            'python%s' % args.python_version,
+            find_executable('rstcheck'),
             '--report', 'warning',
             '--ignore-substitutions', ','.join(ignore_substitutions),
         ] + paths

test/runner/lib/sanity/validate_modules.py

@@ -44,6 +44,7 @@ class ValidateModulesTest(SanitySingleVersion):
             return SanitySkipped(self.name)
 
         cmd = [
+            'python%s' % args.python_version,
             'test/sanity/validate-modules/validate-modules',
             '--format', 'json',
         ] + paths

test/runner/lib/sanity/yamllint.py

@@ -15,6 +15,7 @@ from lib.sanity import (
 from lib.util import (
     SubprocessError,
     run_command,
+    find_executable,
 )
 
 from lib.config import (
@@ -36,7 +37,8 @@ class YamllintTest(SanitySingleVersion):
             return SanitySkipped(self.name)
 
         cmd = [
-            'yamllint',
+            'python%s' % args.python_version,
+            find_executable('yamllint'),
             '--format', 'parsable',
         ] + paths