Ansible Config part2 (#27448)

* Ansible Config part2

- made dump_me nicer, added note this is not prod
- moved internal key removal function to vars
- carry tracebacks in errors we can now show tracebacks for plugins on vvv
- show inventory plugin tracebacks on vvv
- minor fixes to cg groups plugin
- draft config from plugin docs
- made search path warning 'saner' (top level dirs only)
- correctly display config entries and others
- removed unneeded code
- commented out some conn plugin specific from base.yml
- also deprecated sudo/su
- updated ssh conn docs
- shared get option method for connection plugins
- note about needing eval for defaults
- tailored yaml ext
- updated strategy entry
- for connection pliugins, options load on plugin load
- allow for long types in definitions
- better display in ansible-doc
- cleaned up/updated source docs and base.yml
- added many descriptions
- deprecated include toggles as include is
- draft backwards compat get_config
- fixes to ansible-config, added --only-changed
- some code reoorg
- small license headers
- show default in doc type
- pushed module utils details to 5vs
- work w/o config file
- PEPE ATE!
- moved loader to it's own file
- fixed rhn_register test
- fixed boto requirement in make tests
- I ate Pepe
- fixed dynamic eval of defaults
- better doc code

skip ipaddr filter tests when missing netaddr
removed devnull string from config
better becoem resolution

* killed extra space with extreeme prejudice

cause its an affront against all that is holy that 2 spaces touch each other!

shippable timing out on some images, but merging as it passes most

lib/ansible/utils/plugin_docs.py

@@ -20,15 +20,12 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import ast
-import yaml
-
 from collections import MutableMapping, MutableSet, MutableSequence
 
 from ansible.module_utils.six import string_types
-from ansible.parsing.metadata import extract_metadata
+from ansible.parsing.plugin_docs import read_docstring
 from ansible.parsing.yaml.loader import AnsibleLoader
-from ansible.plugins import fragment_loader
+from ansible.plugins.loader import fragment_loader
 
 try:
     from __main__ import display
@@ -93,94 +90,13 @@ def add_fragments(doc, filename):
 
 def get_docstring(filename, verbose=False):
     """
-    Search for assignment of the DOCUMENTATION and EXAMPLES variables
-    in the given file.
-    Parse DOCUMENTATION from YAML and return the YAML doc or None
-    together with EXAMPLES, as plain text.
-
-    DOCUMENTATION can be extended using documentation fragments
-    loaded by the PluginLoader from the module_docs_fragments
-    directory.
+    DOCUMENTATION can be extended using documentation fragments loaded by the PluginLoader from the module_docs_fragments directory.
     """
 
-    # FIXME: Should refactor this so that we have a docstring parsing
-    # function and a separate variable parsing function
-    # Can have a function one higher that invokes whichever is needed
-    #
-    # Should look roughly like this:
-    # get_plugin_doc(filename, verbose=False)
-    #   documentation = extract_docstring(plugin_ast, identifier, verbose=False)
-    #   if not documentation and not (filter or test):
-    #       documentation = extract_variables(plugin_ast)
-    #   documentation['metadata'] = extract_metadata(plugin_ast)
+    data = read_docstring(filename, verbose=verbose)
 
-    data = {
-        'doc': None,
-        'plainexamples': None,
-        'returndocs': None,
-        'metadata': None
-    }
-
-    string_to_vars = {
-        'DOCUMENTATION': 'doc',
-        'EXAMPLES': 'plainexamples',
-        'RETURN': 'returndocs',
-    }
-
-    try:
-        b_module_data = open(filename, 'rb').read()
-        M = ast.parse(b_module_data)
-        try:
-            display.debug('Attempt first docstring is yaml docs')
-            docstring = yaml.load(M.body[0].value.s)
-            for string in string_to_vars.keys():
-                if string in docstring:
-                    data[string_to_vars[string]] = docstring[string]
-                display.debug('assigned :%s' % string_to_vars[string])
-        except Exception as e:
-            display.debug('failed docstring parsing: %s' % str(e))
-
-        if 'docs' not in data or not data['docs']:
-            display.debug('Fallback to vars parsing')
-            for child in M.body:
-                if isinstance(child, ast.Assign):
-                    for t in child.targets:
-                        try:
-                            theid = t.id
-                        except AttributeError:
-                            # skip errors can happen when trying to use the normal code
-                            display.warning("Failed to assign id for %s on %s, skipping" % (t, filename))
-                            continue
-
-                        if theid in string_to_vars:
-                            varkey = string_to_vars[theid]
-                            if isinstance(child.value, ast.Dict):
-                                data[varkey] = ast.literal_eval(child.value)
-                            else:
-                                if theid == 'DOCUMENTATION':
-                                    # string should be yaml
-                                    data[varkey] = AnsibleLoader(child.value.s, file_name=filename).get_single_data()
-                                else:
-                                    # not yaml, should be a simple string
-                                    data[varkey] = child.value.s
-                            display.debug('assigned :%s' % varkey)
-
-        # Metadata is per-file rather than per-plugin/function
-        data['metadata'] = extract_metadata(module_ast=M)[0]
-
-        # add fragments to documentation
-        if data['doc']:
-            add_fragments(data['doc'], filename)
-
-        # remove version
-        if data['metadata']:
-            for x in ('version', 'metadata_version'):
-                if x in data['metadata']:
-                    del data['metadata'][x]
-    except Exception as e:
-        display.error("unable to parse %s" % filename)
-        if verbose is True:
-            display.display("unable to parse %s" % filename)
-            raise
+    # add fragments to documentation
+    if data.get('doc', False):
+        add_fragments(data['doc'], filename)
 
     return data['doc'], data['plainexamples'], data['returndocs'], data['metadata']