Update ansible-test sanity command. (#31958)

* Use correct pip version in ansible-test.
* Add git fallback for validate-modules.
* Run sanity tests in a docker container.
* Use correct python version for sanity tests.
* Pin docker completion images and add default.
* Split pylint execution into multiple contexts.
* Only test .py files in use-argspec-type-path test.
* Accept identical python interpeter name or binary.
* Switch cloud tests to default container.
* Remove unused extras from pip install.
* Filter out empty pip commands.
* Don't force running of pip list.
* Support delegation for windows and network tests.
* Fix ansible-test python version usage.
* Fix ansible-test python version skipping.
* Use absolute path for log in ansible-test.
* Run vyos_command test on python 3.
* Fix windows/network instance persistence.
* Add `test/cache` dir to classification.
* Enable more python versions for network tests.
* Fix cs_router test.

test/runner/lib/pytar.py

@@ -2,76 +2,103 @@
 
 from __future__ import absolute_import, print_function
 
+import abc
 import tarfile
 import os
 
 from lib.util import (
     display,
+    ABC,
 )
 
 # improve performance by disabling uid/gid lookups
 tarfile.pwd = None
 tarfile.grp = None
 
-# To reduce archive time and size, ignore non-versioned files which are large or numerous.
-# Also ignore miscellaneous git related files since the .git directory is ignored.
 
-IGNORE_DIRS = (
-    '.tox',
-    '.git',
-    '.idea',
-    '__pycache__',
-    'ansible.egg-info',
-)
-
-IGNORE_FILES = (
-    '.gitignore',
-    '.gitdir',
-)
-
-IGNORE_EXTENSIONS = (
-    '.pyc',
-    '.retry',
-)
+class TarFilter(ABC):
+    """Filter to use when creating a tar file."""
+    @abc.abstractmethod
+    def ignore(self, item):
+        """
+        :type item: tarfile.TarInfo
+        :rtype: tarfile.TarInfo | None
+        """
+        pass
 
 
-def ignore(item):
+class DefaultTarFilter(TarFilter):
     """
-    :type item: tarfile.TarInfo
-    :rtype: tarfile.TarInfo | None
+    To reduce archive time and size, ignore non-versioned files which are large or numerous.
+    Also ignore miscellaneous git related files since the .git directory is ignored.
     """
-    filename = os.path.basename(item.path)
-    name, ext = os.path.splitext(filename)
-    dirs = os.path.split(item.path)
+    def __init__(self):
+        self.ignore_dirs = (
+            '.tox',
+            '.git',
+            '.idea',
+            '__pycache__',
+            'ansible.egg-info',
+        )
 
-    if not item.isdir():
-        if item.path.startswith('./test/results/'):
+        self.ignore_files = (
+            '.gitignore',
+            '.gitdir',
+        )
+
+        self.ignore_extensions = (
+            '.pyc',
+            '.retry',
+        )
+
+    def ignore(self, item):
+        """
+        :type item: tarfile.TarInfo
+        :rtype: tarfile.TarInfo | None
+        """
+        filename = os.path.basename(item.path)
+        name, ext = os.path.splitext(filename)
+        dirs = os.path.split(item.path)
+
+        if not item.isdir():
+            if item.path.startswith('./test/results/'):
+                return None
+
+            if item.path.startswith('./docs/docsite/_build/'):
+                return None
+
+        if name in self.ignore_files:
             return None
 
-        if item.path.startswith('./docs/docsite/_build/'):
+        if ext in self.ignore_extensions:
             return None
 
-    if name in IGNORE_FILES:
-        return None
+        if any(d in self.ignore_dirs for d in dirs):
+            return None
 
-    if ext in IGNORE_EXTENSIONS:
-        return None
+        return item
 
-    if any(d in IGNORE_DIRS for d in dirs):
-        return None
 
-    return item
+class AllowGitTarFilter(DefaultTarFilter):
+    """
+    Filter that allows git related files normally excluded by the default tar filter.
+    """
+    def __init__(self):
+        super(AllowGitTarFilter, self).__init__()
+
+        self.ignore_dirs = tuple(d for d in self.ignore_dirs if not d.startswith('.git'))
+        self.ignore_files = tuple(f for f in self.ignore_files if not f.startswith('.git'))
 
 
 def create_tarfile(dst_path, src_path, tar_filter):
     """
     :type dst_path: str
     :type src_path: str
-    :type tar_filter: (tarfile.TarInfo) -> tarfile.TarInfo | None
+    :type tar_filter: TarFilter
     """
     display.info('Creating a compressed tar archive of path: %s' % src_path, verbosity=1)
 
     with tarfile.TarFile.gzopen(dst_path, mode='w', compresslevel=4) as tar:
-        tar.add(src_path, filter=tar_filter)
+        tar.add(src_path, filter=tar_filter.ignore)
 
     display.info('Resulting archive is %d bytes.' % os.path.getsize(dst_path), verbosity=1)