Update BOTMETA, fix some plugin authors, improve BOTMETA extra sanity test (#3069)

* Update BOTMETA, fix some plugin authors, improve BOTMETA extra sanity test.

* Linting.

tests/sanity/extra/botmeta.json

@@ -1,8 +1,5 @@
 {
     "include_symlinks": false,
-    "prefixes": [
-        ".github/BOTMETA.yml"
-    ],
     "output": "path-line-column-message",
     "requirements": [
         "PyYAML",

tests/sanity/extra/botmeta.py

@@ -57,8 +57,19 @@ def read_authors(filename):
     return author
 
 
+def extract_author_name(author):
+    m = AUTHOR_REGEX.match(author)
+    if m:
+        return m.group(1)
+    if author == 'Ansible Core Team':
+        return '$team_ansible_core'
+    return None
+
+
 def validate(filename, filedata):
-    if filename.startswith('plugins/doc_fragments/'):
+    if not filename.startswith('plugins/'):
+        return
+    if filename.startswith(('plugins/doc_fragments/', 'plugins/module_utils/')):
         return
     # Compile lis tof all active and inactive maintainers
     all_maintainers = filedata['maintainers'] + filedata['ignore']
@@ -70,21 +81,16 @@ def validate(filename, filedata):
         return
     maintainers = read_authors(filename)
     for maintainer in maintainers:
-        m = AUTHOR_REGEX.match(maintainer)
-        if m:
-            maintainer = m.group(1)
-            if maintainer not in all_maintainers:
-                msg = 'Author %s not mentioned as active or inactive maintainer for %s (mentioned are: %s)' % (
-                    maintainer, filename, ', '.join(all_maintainers))
-                if REPORT_MISSING_MAINTAINERS:
-                    print('%s:%d:%d: %s' % (FILENAME, 0, 0, msg))
+        maintainer = extract_author_name(maintainer)
+        if maintainer is not None and maintainer not in all_maintainers:
+            msg = 'Author %s not mentioned as active or inactive maintainer for %s (mentioned are: %s)' % (
+                maintainer, filename, ', '.join(all_maintainers))
+            if REPORT_MISSING_MAINTAINERS:
+                print('%s:%d:%d: %s' % (FILENAME, 0, 0, msg))
 
 
 def main():
     """Main entry point."""
-    paths = sys.argv[1:] or sys.stdin.read().splitlines()
-    paths = [path for path in paths if path.endswith('/aliases')]
-
     try:
         with open(FILENAME, 'rb') as f:
             botmeta = yaml.safe_load(f)
@@ -100,7 +106,7 @@ def main():
     # Validate schema
 
     MacroSchema = Schema({
-        (str): str,
+        (str): Any(str, None),
     }, extra=PREVENT_EXTRA)
 
     FilesSchema = Schema({
@@ -135,7 +141,11 @@ def main():
 
     def convert_macros(text, macros):
         def f(m):
-            return macros[m.group(1)]
+            macro = m.group(1)
+            replacement = (macros[macro] or '')
+            if macro == 'team_ansible_core':
+                return '$team_ansible_core %s' % replacement
+            return replacement
 
         return macro_re.sub(f, text)
 
@@ -153,31 +163,38 @@ def main():
         return
 
     # Scan all files
-    for dirpath, dirnames, filenames in os.walk('plugins/'):
-        for file in filenames:
-            if file.endswith('.pyc'):
-                continue
-            filename = os.path.join(dirpath, file)
-            if os.path.islink(filename):
-                continue
-            if os.path.isfile(filename):
-                matching_files = []
-                for file, filedata in files.items():
-                    if filename.startswith(file):
-                        matching_files.append((file, filedata))
-                if not matching_files:
-                    print('%s:%d:%d: %s' % (FILENAME, 0, 0, 'Did not find any entry for %s' % filename))
+    unmatched = set(files)
+    for dirs in ('plugins', 'tests', 'changelogs'):
+        for dirpath, dirnames, filenames in os.walk(dirs):
+            for file in sorted(filenames):
+                if file.endswith('.pyc'):
+                    continue
+                filename = os.path.join(dirpath, file)
+                if os.path.islink(filename):
+                    continue
+                if os.path.isfile(filename):
+                    matching_files = []
+                    for file, filedata in files.items():
+                        if filename.startswith(file):
+                            matching_files.append((file, filedata))
+                            if file in unmatched:
+                                unmatched.remove(file)
+                    if not matching_files:
+                        print('%s:%d:%d: %s' % (FILENAME, 0, 0, 'Did not find any entry for %s' % filename))
 
-                matching_files.sort(key=lambda kv: kv[0])
-                filedata = dict()
-                for k in LIST_ENTRIES:
-                    filedata[k] = []
-                for dummy, data in matching_files:
-                    for k, v in data.items():
-                        if k in LIST_ENTRIES:
-                            v = filedata[k] + v
-                        filedata[k] = v
-                validate(filename, filedata)
+                    matching_files.sort(key=lambda kv: kv[0])
+                    filedata = dict()
+                    for k in LIST_ENTRIES:
+                        filedata[k] = []
+                    for dummy, data in matching_files:
+                        for k, v in data.items():
+                            if k in LIST_ENTRIES:
+                                v = filedata[k] + v
+                            filedata[k] = v
+                    validate(filename, filedata)
+
+    for file in unmatched:
+        print('%s:%d:%d: %s' % (FILENAME, 0, 0, 'Entry %s was not used' % file))
 
 
 if __name__ == '__main__':