mirror of
https://github.com/ansible-collections/community.general.git
synced 2025-05-31 05:19:09 -07:00
New metadata 1.0 (#22587)
Changes to the metadata format were approved here: https://github.com/ansible/proposals/issues/54 * Update documentation to the new metadata format * Changes to metadata-tool to account for new metadata * Add GPL license header * Add upgrade subcommand to upgrade metadata version * Change default metadata to the new format * Fix exclusion of non-modules from the metadata report * Fix ansible-doc for new module metadata * Exclude metadata version from ansible-doc output * Fix website docs generation for the new metadata * Update metadata schema in valiate-modules test * Update the metadata in all modules to the new version
This commit is contained in:
Toshio Kuratomi
GitHub
parent
2be3418a81
commit
eb1214baad
1038 changed files with 4298 additions and 3219 deletions
|
|
@ -1,4 +1,24 @@
|
|||
#!/usr/bin/env python
|
||||
# (c) 2016-2017, Toshio Kuratomi <tkuratomi@ansible.com>
|
||||
#
|
||||
# This file is part of Ansible
|
||||
#
|
||||
# Ansible is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Ansible is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Make coding more python3-ish
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import ast
|
||||
import csv
|
||||
|
|
@ -19,7 +39,7 @@ NONMODULE_PY_FILES = frozenset(('async_wrapper.py',))
|
|||
NONMODULE_MODULE_NAMES = frozenset(os.path.splitext(p)[0] for p in NONMODULE_PY_FILES)
|
||||
|
||||
# Default metadata
|
||||
DEFAULT_METADATA = {'version': '1.0', 'status': ['preview'], 'supported_by':'community'}
|
||||
DEFAULT_METADATA = {'metadata_version': '1.0', 'status': ['preview'], 'supported_by':'community'}
|
||||
|
||||
|
||||
class ParseError(Exception):
|
||||
|
|
@ -34,9 +54,10 @@ class MissingModuleError(Exception):
|
|||
|
||||
def usage():
|
||||
print("""Usage:
|
||||
metadata-tester.py report [--version X]
|
||||
metadata-tester.py add [--version X] [--overwrite] CSVFILE
|
||||
metadata-tester.py add-default [--version X] [--overwrite]""")
|
||||
metadata-tool.py report [--version X]
|
||||
metadata-tool.py add [--version X] [--overwrite] CSVFILE
|
||||
metadata-tool.py add-default [--version X] [--overwrite]
|
||||
medatada-tool.py upgrade [--version X]""")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
|
@ -295,7 +316,7 @@ def parse_assigned_metadata_initial(csvfile):
|
|||
:2: Extras (x if so)
|
||||
:3: Category
|
||||
:4: Supported/SLA
|
||||
:5: Committer
|
||||
:5: Curated
|
||||
:6: Stable
|
||||
:7: Deprecated
|
||||
:8: Notes
|
||||
|
|
@ -310,7 +331,7 @@ def parse_assigned_metadata_initial(csvfile):
|
|||
if record[12] == 'core':
|
||||
supported_by = 'core'
|
||||
elif record[12] == 'curated':
|
||||
supported_by = 'committer'
|
||||
supported_by = 'curated'
|
||||
elif record[12] == 'community':
|
||||
supported_by = 'community'
|
||||
else:
|
||||
|
|
@ -326,7 +347,7 @@ def parse_assigned_metadata_initial(csvfile):
|
|||
if not status:
|
||||
status.extend(DEFAULT_METADATA['status'])
|
||||
|
||||
yield (module, {'version': DEFAULT_METADATA['version'], 'supported_by': supported_by, 'status': status})
|
||||
yield (module, {'version': DEFAULT_METADATA['metadata_version'], 'supported_by': supported_by, 'status': status})
|
||||
|
||||
|
||||
def parse_assigned_metadata(csvfile):
|
||||
|
|
@ -334,12 +355,11 @@ def parse_assigned_metadata(csvfile):
|
|||
Fields:
|
||||
:0: Module name
|
||||
:1: supported_by string. One of the valid support fields
|
||||
core, community, unmaintained, committer
|
||||
core, community, curated
|
||||
:2: stableinterface
|
||||
:3: preview
|
||||
:4: deprecated
|
||||
:5: removed
|
||||
:6: tested
|
||||
|
||||
https://github.com/ansible/proposals/issues/30
|
||||
"""
|
||||
|
|
@ -355,12 +375,10 @@ def parse_assigned_metadata(csvfile):
|
|||
status.append('deprecated')
|
||||
if record[5]:
|
||||
status.append('removed')
|
||||
if record[6]:
|
||||
status.append('tested')
|
||||
if not status or record[3]:
|
||||
status.append('preview')
|
||||
|
||||
yield (module, {'version': '1.0', 'supported_by': supported_by, 'status': status})
|
||||
yield (module, {'metadata_version': '1.0', 'supported_by': supported_by, 'status': status})
|
||||
|
||||
|
||||
def write_metadata(filename, new_metadata, version=None, overwrite=False):
|
||||
|
|
@ -388,7 +406,8 @@ def write_metadata(filename, new_metadata, version=None, overwrite=False):
|
|||
|
||||
module_data = insert_metadata(module_data, new_metadata, start_line, targets=('ANSIBLE_METADATA',))
|
||||
|
||||
elif overwrite or (version is not None and ('version' not in current_metadata or StrictVersion(current_metadata['version']) < StrictVersion(version))):
|
||||
elif overwrite or (version is not None and ('metadata_version' not in current_metadata or
|
||||
StrictVersion(current_metadata['metadata_version']) < StrictVersion(version))):
|
||||
# Current metadata that we do not want. Remove the current
|
||||
# metadata and put the new version in its place
|
||||
module_data = remove_metadata(module_data, start_line, start_col, end_line, end_col)
|
||||
|
|
@ -404,7 +423,13 @@ def write_metadata(filename, new_metadata, version=None, overwrite=False):
|
|||
|
||||
|
||||
def return_metadata(plugins):
|
||||
"""Get the metadata for all modules
|
||||
|
||||
Handle duplicate module names
|
||||
|
||||
:arg plugins: List of plugins to look for
|
||||
:returns: Mapping of plugin name to metadata dictionary
|
||||
"""
|
||||
metadata = {}
|
||||
for name, filename in plugins:
|
||||
# There may be several files for a module (if it is written in another
|
||||
|
|
@ -416,6 +441,7 @@ def return_metadata(plugins):
|
|||
metadata[name] = extract_metadata(module_data)[0]
|
||||
return metadata
|
||||
|
||||
|
||||
def metadata_summary(plugins, version=None):
|
||||
"""Compile information about the metadata status for a list of modules
|
||||
|
||||
|
|
@ -431,8 +457,8 @@ def metadata_summary(plugins, version=None):
|
|||
has_metadata = {}
|
||||
supported_by = defaultdict(set)
|
||||
status = defaultdict(set)
|
||||
requested_version = StrictVersion(version)
|
||||
|
||||
plugins = list(plugins)
|
||||
all_mods_metadata = return_metadata(plugins)
|
||||
for name, filename in plugins:
|
||||
# Does the module have metadata?
|
||||
|
|
@ -440,7 +466,7 @@ def metadata_summary(plugins, version=None):
|
|||
metadata = all_mods_metadata[name]
|
||||
if metadata is None:
|
||||
no_metadata[name] = filename
|
||||
elif version is not None and ('version' not in metadata or StrictVersion(metadata['version']) < StrictVersion(version)):
|
||||
elif version is not None and ('metadata_version' not in metadata or StrictVersion(metadata['metadata_version']) < requested_version):
|
||||
no_metadata[name] = filename
|
||||
else:
|
||||
has_metadata[name] = filename
|
||||
|
|
@ -457,6 +483,34 @@ def metadata_summary(plugins, version=None):
|
|||
|
||||
return list(no_metadata.values()), list(has_metadata.values()), supported_by, status
|
||||
|
||||
#
|
||||
# Filters to convert between metadata versions
|
||||
#
|
||||
|
||||
def convert_metadata_pre_1_0_to_1_0(metadata):
|
||||
"""
|
||||
Convert pre-1.0 to 1.0 metadata format
|
||||
|
||||
:arg metadata: The old metadata
|
||||
:returns: The new metadata
|
||||
|
||||
Changes from pre-1.0 to 1.0:
|
||||
* ``version`` field renamed to ``metadata_version``
|
||||
* ``supported_by`` field value ``unmaintained`` has been removed (change to
|
||||
``community`` and let an external list track whether a module is unmaintained)
|
||||
* ``supported_by`` field value ``committer`` has been renamed to ``curated``
|
||||
"""
|
||||
new_metadata = {'metadata_version': '1.0',
|
||||
'supported_by': metadata['supported_by'],
|
||||
'status': metadata['status']
|
||||
}
|
||||
if new_metadata['supported_by'] == 'unmaintained':
|
||||
new_metadata['supported_by'] = 'community'
|
||||
elif new_metadata['supported_by'] == 'committer':
|
||||
new_metadata['supported_by'] = 'curated'
|
||||
|
||||
return new_metadata
|
||||
|
||||
#
|
||||
# Subcommands
|
||||
#
|
||||
|
|
@ -515,6 +569,62 @@ def add_default(version=None, overwrite=False):
|
|||
return 0
|
||||
|
||||
|
||||
def upgrade_metadata(version=None):
|
||||
"""Implement the subcommand to upgrade the default metadata in modules.
|
||||
|
||||
:kwarg version: If given, the version of the metadata to upgrade to. If
|
||||
not given, upgrade to the latest format version.
|
||||
"""
|
||||
if version is None:
|
||||
# Number larger than any of the defined metadata formats.
|
||||
version = 9999999
|
||||
requested_version = StrictVersion(version)
|
||||
|
||||
# List all plugins
|
||||
plugins = module_loader.all(path_only=True)
|
||||
plugins = ((os.path.splitext((os.path.basename(p)))[0], p) for p in plugins)
|
||||
plugins = (p for p in plugins if p[0] not in NONMODULE_MODULE_NAMES)
|
||||
|
||||
processed = set()
|
||||
diagnostic_messages = []
|
||||
for name, filename in (info for info in plugins if info[0] not in processed):
|
||||
# For each plugin, read the existing metadata
|
||||
with open(filename, 'rb') as f:
|
||||
module_data = f.read()
|
||||
metadata = extract_metadata(module_data)[0]
|
||||
|
||||
# If the metadata isn't the requested version, convert it to the new
|
||||
# version
|
||||
if 'metadata_version' not in metadata or metadata['metadata_version'] != version:
|
||||
#
|
||||
# With each iteration of metadata, add a new conditional to
|
||||
# upgrade from the previous version
|
||||
#
|
||||
|
||||
if 'metadata_version' not in metadata:
|
||||
# First version, pre-1.0 final metadata
|
||||
metadata = convert_metadata_pre_1_0_to_1_0(metadata)
|
||||
|
||||
if metadata['metadata_version'] == '1.0' and StrictVersion('1.0') < requested_version:
|
||||
# 1.0 version => XXX. We don't yet have anything beyond 1.0
|
||||
# so there's nothing here
|
||||
pass
|
||||
|
||||
# Replace the existing metadata with the new format
|
||||
try:
|
||||
write_metadata(filename, metadata, version, overwrite=True)
|
||||
except ParseError as e:
|
||||
diagnostic_messages.append(e.args[0])
|
||||
continue
|
||||
|
||||
processed.add(name)
|
||||
|
||||
if diagnostic_messages:
|
||||
pprint(diagnostic_messages)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def report(version=None):
|
||||
"""Implement the report subcommand
|
||||
|
||||
|
|
@ -525,9 +635,8 @@ def report(version=None):
|
|||
"""
|
||||
# List of all plugins
|
||||
plugins = module_loader.all(path_only=True)
|
||||
plugins = list(plugins)
|
||||
plugins = ((os.path.splitext((os.path.basename(p)))[0], p) for p in plugins)
|
||||
plugins = (p for p in plugins if p[0] != NONMODULE_MODULE_NAMES)
|
||||
plugins = (p for p in plugins if p[0] not in NONMODULE_MODULE_NAMES)
|
||||
plugins = list(plugins)
|
||||
|
||||
no_metadata, has_metadata, support, status = metadata_summary(plugins, version=version)
|
||||
|
|
@ -542,8 +651,8 @@ def report(version=None):
|
|||
|
||||
print('== Supported by core ==')
|
||||
pprint(sorted(support['core']))
|
||||
print('== Supported by committers ==')
|
||||
pprint(sorted(support['committer']))
|
||||
print('== Supported by value curated ==')
|
||||
pprint(sorted(support['curated']))
|
||||
print('== Supported by community ==')
|
||||
pprint(sorted(support['community']))
|
||||
print('')
|
||||
|
|
@ -560,12 +669,10 @@ def report(version=None):
|
|||
|
||||
print('== Summary ==')
|
||||
print('No Metadata: {0} Has Metadata: {1}'.format(len(no_metadata), len(has_metadata)))
|
||||
print('Supported by core: {0} Supported by community: {1} Supported by committer: {2}'.format(len(support['core']), len(support['community']),
|
||||
len(support['committer'])))
|
||||
print('Supported by core: {0} Supported by community: {1} Supported by value curated: {2}'.format(len(support['core']),
|
||||
len(support['community']), len(support['curated'])))
|
||||
print('Status StableInterface: {0} Status Preview: {1} Status Deprecated: {2} Status Removed: {3}'.format(len(status['stableinterface']),
|
||||
len(status['preview']),
|
||||
len(status['deprecated']),
|
||||
len(status['removed'])))
|
||||
len(status['preview']), len(status['deprecated']), len(status['removed'])))
|
||||
|
||||
return 0
|
||||
|
||||
|
|
@ -573,13 +680,13 @@ def report(version=None):
|
|||
if __name__ == '__main__':
|
||||
action, args = parse_args(sys.argv[1:])
|
||||
|
||||
### TODO: Implement upgrade metadata and upgrade metadata from csvfile
|
||||
if action == 'report':
|
||||
rc = report(version=args['version'])
|
||||
elif action == 'add':
|
||||
rc = add_from_csv(args['csvfile'], version=args['version'], overwrite=args['overwrite'])
|
||||
elif action == 'add-default':
|
||||
rc = add_default(version=args['version'], overwrite=args['overwrite'])
|
||||
elif action == 'upgrade':
|
||||
rc = upgrade_metadata(version=args['version'])
|
||||
|
||||
sys.exit(rc)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue