mirror of
https://github.com/ansible-collections/community.general.git
synced 2025-04-25 11:51:26 -07:00
more updates to plugin/config generation (#30787)
* fixed module generation added missing lookup page point to plugins when plugins made modules singular add display for verbose an debug messages nicer templating, changed generation order for ref corrected links moved most of lookup docs to plugin section * Copy edits * Fixed typos * Clarified wording
This commit is contained in:
Brian Coca
GitHub
parent
3886f80e44
commit
2ed46e04f4
16 changed files with 345 additions and 136 deletions
|
|
@ -26,7 +26,7 @@ import datetime
|
|||
import glob
|
||||
import optparse
|
||||
import os
|
||||
import pprint
|
||||
from pprint import PrettyPrinter
|
||||
import re
|
||||
import sys
|
||||
import warnings
|
||||
|
|
@ -47,6 +47,7 @@ from six import iteritems, string_types
|
|||
from ansible.errors import AnsibleError
|
||||
from ansible.module_utils._text import to_bytes
|
||||
from ansible.utils import plugin_docs
|
||||
from ansible.utils.display import Display
|
||||
|
||||
|
||||
#####################################################################################
|
||||
|
|
@ -74,6 +75,9 @@ _CONST = re.compile(r"C\(([^)]+)\)")
|
|||
|
||||
DEPRECATED = b" (D)"
|
||||
|
||||
pp = PrettyPrinter()
|
||||
display = Display()
|
||||
|
||||
|
||||
def rst_ify(text):
|
||||
''' convert symbols like I(this is in italics) to valid restructured text '''
|
||||
|
|
@ -132,13 +136,13 @@ def write_data(text, output_dir, outputname, module=None):
|
|||
print(text)
|
||||
|
||||
|
||||
def get_module_info(module_dir, limit_to_modules=None, verbose=False):
|
||||
def get_plugin_info(module_dir, limit_to=None, verbose=False):
|
||||
'''
|
||||
Returns information about modules and the categories that they belong to
|
||||
Returns information about plugins and the categories that they belong to
|
||||
|
||||
:arg module_dir: file system path to the top of the modules directory
|
||||
:kwarg limit_to_modules: If given, this is a list of module names to
|
||||
generate information for. All other modules will be ignored.
|
||||
:arg module_dir: file system path to the top of the plugin directory
|
||||
:kwarg limit_to: If given, this is a list of plugin names to
|
||||
generate information for. All other plugins will be ignored.
|
||||
:returns: Tuple of two dicts containing module_info, categories, and
|
||||
aliases and a set listing deprecated modules:
|
||||
|
||||
|
|
@ -184,7 +188,7 @@ def get_module_info(module_dir, limit_to_modules=None, verbose=False):
|
|||
|
||||
# If requested, limit module documentation building only to passed-in
|
||||
# modules.
|
||||
if limit_to_modules is not None and module.lower() not in limit_to_modules:
|
||||
if limit_to is not None and module.lower() not in limit_to:
|
||||
continue
|
||||
|
||||
deprecated = False
|
||||
|
|
@ -260,15 +264,15 @@ def generate_parser():
|
|||
|
||||
p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
|
||||
p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
|
||||
p.add_option("-P", "--plugin-type", action="store", dest="plugin_type", default='modules', help="The type of plugin (plugins, modules)")
|
||||
p.add_option("-P", "--plugin-type", action="store", dest="plugin_type", default='module', help="The type of plugin (module, lookup, etc)")
|
||||
p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates")
|
||||
p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type")
|
||||
p.add_option("-v", "--verbose", action='store_true', default=False, help="Verbose")
|
||||
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
|
||||
p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules")
|
||||
p.add_option("-l", "--limit-to-modules", action="store", dest="limit_to_modules", default=None,
|
||||
help="Limit building module documentation to comma-separated list of modules. Specify non-existing module name for no modules.")
|
||||
p.add_option("-l", "--limit-to-modules", '--limit-to', action="store", dest="limit_to", default=None,
|
||||
help="Limit building module documentation to comma-separated list of plugins. Specify non-existing plugin name for no plugins.")
|
||||
p.add_option('-V', action='version', help='Show version number and exit')
|
||||
p.add_option('-v', '--verbose', dest='verbosity', default=0, action="count", help="verbose mode (increase number of 'v's for more)")
|
||||
return p
|
||||
|
||||
|
||||
|
|
@ -287,11 +291,17 @@ def jinja2_environment(template_dir, typ, plugin_type):
|
|||
env.filters['fmt'] = rst_fmt
|
||||
env.filters['xline'] = rst_xline
|
||||
templates['plugin'] = env.get_template('plugin.rst.j2')
|
||||
templates['category_list'] = env.get_template('%s_by_category.rst.j2' % plugin_type)
|
||||
templates['support_list'] = env.get_template('%s_by_support.rst.j2' % plugin_type)
|
||||
templates['list_of_CATEGORY_modules'] = env.get_template('list_of_CATEGORY_%s.rst.j2' % plugin_type)
|
||||
|
||||
if plugin_type == 'module':
|
||||
name = 'modules'
|
||||
else:
|
||||
name = 'plugins'
|
||||
|
||||
templates['category_list'] = env.get_template('%s_by_category.rst.j2' % name)
|
||||
templates['support_list'] = env.get_template('%s_by_support.rst.j2' % name)
|
||||
templates['list_of_CATEGORY_modules'] = env.get_template('list_of_CATEGORY_%s.rst.j2' % name)
|
||||
else:
|
||||
raise Exception("unknown module format type: %s" % typ)
|
||||
raise Exception("Unsupported format type: %s" % typ)
|
||||
|
||||
return templates
|
||||
|
||||
|
|
@ -309,22 +319,17 @@ def too_old(added):
|
|||
return added_float < TO_OLD_TO_BE_NOTABLE
|
||||
|
||||
|
||||
def process_modules(module_map, templates, outputname, output_dir, ansible_version, plugin_type):
|
||||
def process_plugins(module_map, templates, outputname, output_dir, ansible_version, plugin_type):
|
||||
for module in module_map:
|
||||
# print("rendering: %s" % module)
|
||||
|
||||
# pprint.pprint(('process_modules module:', module))
|
||||
|
||||
display.display("rendering: %s" % module)
|
||||
fname = module_map[module]['path']
|
||||
|
||||
# pprint.pprint(('process_modules module_info: ', module_map[module]))
|
||||
|
||||
module_categories = module_map[module].get('categories', [])
|
||||
display.vvvvv(pp.pformat(('process_plugins info: ', module_map[module])))
|
||||
|
||||
# crash if module is missing documentation and not explicitly hidden from docs index
|
||||
if module_map[module]['doc'] is None:
|
||||
print("%s: ERROR: MODULE MISSING DOCUMENTATION" % (fname,))
|
||||
_doc = {'module': module,
|
||||
display.error("%s MISSING DOCUMENTATION" % (fname,))
|
||||
_doc = {plugin_type: module,
|
||||
'version_added': '2.4',
|
||||
'filename': fname}
|
||||
module_map[module]['doc'] = _doc
|
||||
|
|
@ -332,8 +337,7 @@ def process_modules(module_map, templates, outputname, output_dir, ansible_versi
|
|||
|
||||
# Going to reference this heavily so make a short name to reference it by
|
||||
doc = module_map[module]['doc']
|
||||
|
||||
# pprint.pprint(('process_modules doc: ', doc))
|
||||
display.vvvvv(pp.pformat(('process_plugins doc: ', doc)))
|
||||
|
||||
# add some defaults for plugins that dont have most of the info
|
||||
doc['module'] = doc.get('module', module)
|
||||
|
|
@ -342,12 +346,12 @@ def process_modules(module_map, templates, outputname, output_dir, ansible_versi
|
|||
doc['plugin_type'] = plugin_type
|
||||
|
||||
if module_map[module]['deprecated'] and 'deprecated' not in doc:
|
||||
print("%s: WARNING: MODULE MISSING DEPRECATION DOCUMENTATION: %s" % (fname, 'deprecated'))
|
||||
display.warning("%s PLUGIN MISSING DEPRECATION DOCUMENTATION: %s" % (fname, 'deprecated'))
|
||||
|
||||
required_fields = ('short_description',)
|
||||
for field in required_fields:
|
||||
if field not in doc:
|
||||
print("%s: WARNING: MODULE MISSING field '%s'" % (fname, field))
|
||||
display.warning("%s PLUGIN MISSING field '%s'" % (fname, field))
|
||||
|
||||
not_nullable_fields = ('short_description',)
|
||||
for field in not_nullable_fields:
|
||||
|
|
@ -355,8 +359,7 @@ def process_modules(module_map, templates, outputname, output_dir, ansible_versi
|
|||
print("%s: WARNING: MODULE field '%s' DOCUMENTATION is null/empty value=%s" % (fname, field, doc[field]))
|
||||
|
||||
if 'version_added' not in doc:
|
||||
pprint.pprint(doc)
|
||||
# sys.exit("*** ERROR: missing version_added in: %s ***\n" % module)
|
||||
display.error("*** ERROR: missing version_added in: %s ***\n" % module)
|
||||
|
||||
#
|
||||
# The present template gets everything from doc so we spend most of this
|
||||
|
|
@ -424,7 +427,7 @@ def process_modules(module_map, templates, outputname, output_dir, ansible_versi
|
|||
|
||||
doc['metadata'] = module_map[module]['metadata']
|
||||
|
||||
# pprint.pprint(module_map[module]
|
||||
display.vvvvv(pp.pformat(module_map[module]))
|
||||
if module_map[module]['returndocs']:
|
||||
try:
|
||||
doc['returndocs'] = yaml.safe_load(module_map[module]['returndocs'])
|
||||
|
|
@ -438,29 +441,18 @@ def process_modules(module_map, templates, outputname, output_dir, ansible_versi
|
|||
if isinstance(doc['author'], string_types):
|
||||
doc['author'] = [doc['author']]
|
||||
|
||||
# print('about to template')
|
||||
# pprint.pprint(doc)
|
||||
display.v('about to template %s' % module)
|
||||
display.vvvvv(pp.pformat(doc))
|
||||
text = templates['plugin'].render(doc)
|
||||
|
||||
# plugins get namespace dirs but modules do not
|
||||
if plugin_type == 'plugins':
|
||||
for module_category in module_categories:
|
||||
category_output_dir = os.path.join(output_dir, 'plugins', '%s' % module_category)
|
||||
write_data(text, category_output_dir, outputname, module)
|
||||
else:
|
||||
write_data(text, output_dir, outputname, module)
|
||||
write_data(text, output_dir, outputname, module)
|
||||
|
||||
|
||||
def process_categories(mod_info, categories, templates,
|
||||
output_dir, output_name, plugin_type):
|
||||
def process_categories(plugin_info, categories, templates, output_dir, output_name, plugin_type):
|
||||
for category in sorted(categories.keys()):
|
||||
if (plugin_type, category) == ('plugins', ''):
|
||||
print('skipping unknown cat: %s' % category)
|
||||
continue
|
||||
module_map = categories[category]
|
||||
category_filename = output_name % category
|
||||
|
||||
print("*** recording category %s in %s ***" % (category, category_filename))
|
||||
display.display("*** recording category %s in %s ***" % (category, category_filename))
|
||||
|
||||
# start a new category file
|
||||
|
||||
|
|
@ -472,7 +464,7 @@ def process_categories(mod_info, categories, templates,
|
|||
'category_name': category_name,
|
||||
'category': module_map,
|
||||
'subcategories': subcategories,
|
||||
'module_info': mod_info,
|
||||
'module_info': plugin_info,
|
||||
'plugin_type': plugin_type
|
||||
}
|
||||
|
||||
|
|
@ -480,7 +472,7 @@ def process_categories(mod_info, categories, templates,
|
|||
write_data(text, output_dir, category_filename)
|
||||
|
||||
|
||||
def process_support_levels(mod_info, templates, output_dir, plugin_type):
|
||||
def process_support_levels(plugin_info, templates, output_dir, plugin_type):
|
||||
supported_by = {'Ansible Core Team': {'slug': 'core_supported',
|
||||
'modules': [],
|
||||
'output': 'core_maintained.rst',
|
||||
|
|
@ -528,9 +520,9 @@ These modules are currently shipped with Ansible, but will most likely be shippe
|
|||
if plugin_type == 'plugins':
|
||||
return
|
||||
# Separate the modules by support_level
|
||||
for module, info in mod_info.items():
|
||||
for module, info in plugin_info.items():
|
||||
if not info.get('metadata', None):
|
||||
print('no metadata for %s' % module)
|
||||
display.warning('no metadata for %s' % module)
|
||||
continue
|
||||
if info['metadata']['supported_by'] == 'core':
|
||||
supported_by['Ansible Core Team']['modules'].append(module)
|
||||
|
|
@ -548,7 +540,7 @@ These modules are currently shipped with Ansible, but will most likely be shippe
|
|||
template_data = {'maintainers': maintainers,
|
||||
'modules': data['modules'],
|
||||
'slug': data['slug'],
|
||||
'module_info': mod_info,
|
||||
'module_info': plugin_info,
|
||||
}
|
||||
text = templates['support_list'].render(template_data)
|
||||
write_data(text, output_dir, data['output'])
|
||||
|
|
@ -567,62 +559,72 @@ def validate_options(options):
|
|||
|
||||
def main():
|
||||
|
||||
# INIT
|
||||
p = generate_parser()
|
||||
|
||||
(options, args) = p.parse_args()
|
||||
validate_options(options)
|
||||
|
||||
display.verbosity = options.verbosity
|
||||
plugin_type = options.plugin_type
|
||||
|
||||
if plugin_type == 'modules':
|
||||
templates = jinja2_environment(options.template_dir, options.type, plugin_type)
|
||||
output_dir = options.output_dir
|
||||
# prep templating
|
||||
templates = jinja2_environment(options.template_dir, options.type, plugin_type)
|
||||
|
||||
# set file/directory structure
|
||||
if plugin_type == 'module':
|
||||
# trim trailing s off of plugin_type for plugin_type=='modules'. ie 'copy_module.rst'
|
||||
outputname = '%s_' + '%s.rst' % plugin_type[:-1]
|
||||
outputname = '%s_' + '%s.rst' % plugin_type
|
||||
output_dir = options.output_dir
|
||||
else:
|
||||
templates = jinja2_environment(options.template_dir, options.type, 'plugins')
|
||||
# for plugins, just use 'ssh.rst' vs 'ssh_module.rst'
|
||||
outputname = '%s.rst'
|
||||
output_dir = '%s/plugins/%s' % (options.output_dir, plugin_type)
|
||||
|
||||
# Convert passed-in limit_to_modules to None or list of modules.
|
||||
if options.limit_to_modules is not None:
|
||||
options.limit_to_modules = [s.lower() for s in options.limit_to_modules.split(",")]
|
||||
display.vv('output name: %s' % outputname)
|
||||
display.vv('output dir: %s' % output_dir)
|
||||
|
||||
mod_info, categories = get_module_info(options.module_dir, limit_to_modules=options.limit_to_modules, verbose=options.verbose)
|
||||
# Convert passed-in limit_to to None or list of modules.
|
||||
if options.limit_to is not None:
|
||||
options.limit_to = [s.lower() for s in options.limit_to.split(",")]
|
||||
|
||||
categories['all'] = {'_modules': mod_info.keys()}
|
||||
plugin_info, categories = get_plugin_info(options.module_dir, limit_to=options.limit_to, verbose=(options.verbosity > 0))
|
||||
|
||||
categories['all'] = {'_modules': plugin_info.keys()}
|
||||
|
||||
display.vvv(pp.pformat(categories))
|
||||
display.vvvvv(pp.pformat(plugin_info))
|
||||
|
||||
# pprint.pprint(categories)
|
||||
# pprint.pprint(mod_info)
|
||||
# pprint.pprint(dict(mod_info))
|
||||
# Transform the data
|
||||
if options.type == 'rst':
|
||||
for key, record in mod_info.items():
|
||||
# pprint.pprint(('record', record))
|
||||
display.v('Generating rst')
|
||||
for key, record in plugin_info.items():
|
||||
display.vv(key)
|
||||
display.vvvvv(pp.pformat(('record', record)))
|
||||
if record.get('doc', None):
|
||||
short_desc = record['doc']['short_description']
|
||||
if short_desc is None:
|
||||
print('WARNING: short_description for %s is None' % key)
|
||||
display.warning('short_description for %s is None' % key)
|
||||
short_desc = ''
|
||||
record['doc']['short_description'] = rst_ify(short_desc)
|
||||
|
||||
if plugin_type == 'modules':
|
||||
# Write master category list
|
||||
if plugin_type == 'module':
|
||||
display.v('Generating Categories')
|
||||
# Write module master category list
|
||||
category_list_text = templates['category_list'].render(categories=sorted(categories.keys()))
|
||||
category_index_name = '%s_by_category.rst' % plugin_type
|
||||
category_index_name = '%ss_by_category.rst' % plugin_type
|
||||
write_data(category_list_text, output_dir, category_index_name)
|
||||
|
||||
# Render all the individual module pages
|
||||
process_modules(mod_info, templates, outputname, output_dir, options.ansible_version, plugin_type)
|
||||
# Render all the individual plugin pages
|
||||
display.v('Generating plugin pages')
|
||||
process_plugins(plugin_info, templates, outputname, output_dir, options.ansible_version, plugin_type)
|
||||
|
||||
# Render all the categories for modules
|
||||
if plugin_type == 'modules':
|
||||
category_list_name_template = 'list_of_%s_' + '%s.rst' % plugin_type
|
||||
process_categories(mod_info, categories, templates, output_dir, category_list_name_template, plugin_type)
|
||||
if plugin_type == 'module':
|
||||
display.v('Generating Category lists')
|
||||
category_list_name_template = 'list_of_%s_' + '%ss.rst' % plugin_type
|
||||
process_categories(plugin_info, categories, templates, output_dir, category_list_name_template, plugin_type)
|
||||
|
||||
# Render all the categories for modules
|
||||
process_support_levels(mod_info, templates, output_dir, plugin_type)
|
||||
process_support_levels(plugin_info, templates, output_dir, plugin_type)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue