Enable documentation in plugins

Made ansible-doc more plugin agnostic
We can have docs in lookup, callback, connectionm strategy, etc
Use first docstring and make pepizis happy
generalized module_docs to plugin_docs
documented cartesian, ssh, default, jsonfile, etc as examples
changed lack of docs to warning when listing
made smarter about bad docstrings
better blacklisting
added handling of options/config/envs/etc
move blacklist to find_plugins, only need once

hacking/module_formatter.py

@@ -37,7 +37,7 @@ from six import iteritems
 
 from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes
-from ansible.utils import module_docs
+from ansible.utils import plugin_docs
 
 #####################################################################################
 # constants and paths
@@ -158,7 +158,7 @@ def list_modules(module_dir, depth=0):
             category = category[new_cat]
 
         module = os.path.splitext(os.path.basename(module_path))[0]
-        if module in module_docs.BLACKLIST_MODULES:
+        if module in plugin_docs.BLACKLIST['MODULE']:
             # Do not list blacklisted modules
             continue
         if module.startswith("_") and os.path.islink(module_path):
@@ -254,7 +254,7 @@ def process_module(module, options, env, template, outputname, module_map, alias
     print("rendering: %s" % module)
 
     # use ansible core library to parse out doc metadata YAML and plaintext examples
-    doc, examples, returndocs, metadata = module_docs.get_docstring(fname, verbose=options.verbose)
+    doc, examples, returndocs, metadata = plugin_docs.get_docstring(fname, verbose=options.verbose)
 
     # crash if module is missing documentation and not explicitly hidden from docs index
     if doc is None:

lib/ansible/cli/console.py

@@ -46,7 +46,7 @@ from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.splitter import parse_kv
 from ansible.playbook.play import Play
 from ansible.plugins import module_loader
-from ansible.utils import module_docs
+from ansible.utils import plugin_docs
 from ansible.utils.color import stringc
 from ansible.vars import VariableManager
 
@@ -356,7 +356,7 @@ class ConsoleCLI(CLI, cmd.Cmd):
         if module_name in self.modules:
             in_path = module_loader.find_plugin(module_name)
             if in_path:
-                oc, a, _, _ = module_docs.get_docstring(in_path)
+                oc, a, _, _ = plugin_docs.get_docstring(in_path)
                 if oc:
                     display.display(oc['short_description'])
                     display.display('Parameters:')
@@ -388,7 +388,7 @@ class ConsoleCLI(CLI, cmd.Cmd):
 
     def module_args(self, module_name):
         in_path = module_loader.find_plugin(module_name)
-        oc, a, _, _ = module_docs.get_docstring(in_path)
+        oc, a, _, _ = plugin_docs.get_docstring(in_path)
         return list(oc['options'].keys())
 
     def run(self):

lib/ansible/cli/doc.py

@@ -23,14 +23,15 @@ import datetime
 import os
 import traceback
 import textwrap
+import yaml
 
 from ansible.compat.six import iteritems, string_types
 
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleOptionsError
-from ansible.plugins import module_loader, action_loader
+from ansible.plugins import module_loader, action_loader, lookup_loader, callback_loader, cache_loader, connection_loader, strategy_loader
 from ansible.cli import CLI
-from ansible.utils import module_docs
+from ansible.utils import plugin_docs
 
 try:
     from __main__ import display
@@ -40,27 +41,29 @@ except ImportError:
 
 
 class DocCLI(CLI):
-    """ Vault command line class """
+    """ Doc command line class """
 
     def __init__(self, args):
 
         super(DocCLI, self).__init__(args)
-        self.module_list = []
+        self.plugin_list = set()
 
     def parse(self):
 
         self.parser = CLI.base_parser(
-            usage='usage: %prog [options] [module...]',
+            usage='usage: %prog [options] [plugin ...]',
-            epilog='Show Ansible module documentation',
+            epilog='Show Ansible plugin documentation',
             module_opts=True,
         )
 
         self.parser.add_option("-l", "--list", action="store_true", default=False, dest='list_dir',
-                help='List available modules')
+                help='List available plugins')
         self.parser.add_option("-s", "--snippet", action="store_true", default=False, dest='show_snippet',
-                help='Show playbook snippet for specified module(s)')
+                help='Show playbook snippet for specified plugin(s)')
-        self.parser.add_option("-a", "--all", action="store_true", default=False, dest='all_modules',
+        self.parser.add_option("-a", "--all", action="store_true", default=False, dest='all_plugins',
-                help='Show documentation for all modules')
+                help='Show documentation for all plugins')
+        self.parser.add_option("-t", "--type", action="store", default='module', dest='type', type='choice',
+                help='Choose which plugin type', choices=['module','cache', 'connection', 'callback', 'lookup', 'strategy'])
 
         super(DocCLI, self).parse()
 
@@ -70,76 +73,85 @@ class DocCLI(CLI):
 
         super(DocCLI, self).run()
 
+        plugin_type = self.options.type
+
+        # choose plugin type
+        if plugin_type == 'cache':
+            loader = cache_loader
+        elif plugin_type == 'callback':
+            loader = callback_loader
+        elif plugin_type == 'connection':
+            loader = connection_loader
+        elif plugin_type == 'lookup':
+            loader = lookup_loader
+        elif plugin_type == 'strategy':
+            loader = strategy_loader
+        else:
+            loader = module_loader
+
+        # add to plugin path from command line
         if self.options.module_path is not None:
             for i in self.options.module_path.split(os.pathsep):
-                module_loader.add_directory(i)
+                loader.add_directory(i)
 
-        # list modules
+        # list plugins for type
         if self.options.list_dir:
-            paths = module_loader._get_paths()
+            paths = loader._get_paths()
             for path in paths:
-                self.find_modules(path)
+                self.find_plugins(path, plugin_type)
 
-            self.pager(self.get_module_list_text())
+            self.pager(self.get_plugin_list_text(loader))
             return 0
 
-        # process all modules
+        # process all plugins of type
-        if self.options.all_modules:
+        if self.options.all_plugins:
-            paths = module_loader._get_paths()
+            paths = loader._get_paths()
             for path in paths:
-                self.find_modules(path)
+                self.find_plugins(path, plugin_type)
-            self.args = sorted(set(self.module_list) - module_docs.BLACKLIST_MODULES)
 
         if len(self.args) == 0:
             raise AnsibleOptionsError("Incorrect options passed")
 
-        # process command line module list
+        # process command line list
         text = ''
-        for module in self.args:
+        for plugin in self.args:
 
             try:
-                # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
+                # if the plugin lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
-                filename = module_loader.find_plugin(module, mod_type='.py', ignore_deprecated=True)
+                filename = loader.find_plugin(plugin, mod_type='.py', ignore_deprecated=True)
                 if filename is None:
-                    display.warning("module %s not found in %s\n" % (module, DocCLI.print_paths(module_loader)))
+                    display.warning("%s %s not found in %s\n" % (plugin_type, plugin, DocCLI.print_paths(loader)))
                     continue
 
                 if any(filename.endswith(x) for x in C.BLACKLIST_EXTS):
                     continue
 
                 try:
-                    doc, plainexamples, returndocs, metadata = module_docs.get_docstring(filename, verbose=(self.options.verbosity > 0))
+                    doc, plainexamples, returndocs, metadata = plugin_docs.get_docstring(filename, verbose=(self.options.verbosity > 0))
                 except:
                     display.vvv(traceback.format_exc())
-                    display.error("module %s has a documentation error formatting or is missing documentation\nTo see exact traceback use -vvv" % module)
+                    display.error("%s %s has a documentation error formatting or is missing documentation." % (plugin_type, plugin))
                     continue
 
                 if doc is not None:
 
-                    # is there corresponding action plugin?
+                    # assign from other sections
-                    if module in action_loader:
-                        doc['action'] = True
-                    else:
-                        doc['action'] = False
-
-                    all_keys = []
-                    for (k,v) in iteritems(doc['options']):
-                        all_keys.append(k)
-                    all_keys = sorted(all_keys)
-                    doc['option_keys'] = all_keys
-
-                    doc['filename']         = filename
-                    doc['docuri']           = doc['module'].replace('_', '-')
-                    doc['now_date']         = datetime.date.today().strftime('%Y-%m-%d')
                     doc['plainexamples']    = plainexamples
                     doc['returndocs']       = returndocs
                     doc['metadata']         = metadata
-                    if 'metadata_version' in doc['metadata']:
-                        del doc['metadata']['metadata_version']
-                    if 'version' in doc['metadata']:
-                        del doc['metadata']['metadata_version']
 
-                    if self.options.show_snippet:
+                    # generate extra data
+                    if plugin_type == 'module':
+                        # is there corresponding action plugin?
+                        if plugin in action_loader:
+                            doc['action'] = True
+                        else:
+                            doc['action'] = False
+                    doc['filename']         = filename
+                    doc['now_date']         = datetime.date.today().strftime('%Y-%m-%d')
+                    doc['docuri']           = doc[plugin_type].replace('_', '-')
+
+                    if self.options.show_snippet and plugin_type == 'module':
                         text += self.get_snippet_text(doc)
                     else:
                         text += self.get_man_text(doc)
@@ -149,47 +161,56 @@ class DocCLI(CLI):
                     raise AnsibleError("Parsing produced an empty object.")
             except Exception as e:
                 display.vvv(traceback.format_exc())
-                raise AnsibleError("module %s missing documentation (or could not parse documentation): %s\n" % (module, str(e)))
+                raise AnsibleError("%s %s missing documentation (or could not parse documentation): %s\n" % (plugin_type, plugin, str(e)))
 
         if text:
             self.pager(text)
         return 0
 
-    def find_modules(self, path):
+    def find_plugins(self, path, ptype):
-        for module in os.listdir(path):
-            full_path = '/'.join([path, module])
 
-            if module.startswith('.'):
+        display.vvvv("Searching %s for plugins" % path)
+
+        if not os.path.exists(path):
+            display.vvvv("%s does not exist" % path)
+            return
+
+        bkey = ptype.upper()
+        for plugin in os.listdir(path):
+            display.vvvv("Found %s" % plugin)
+            full_path = '/'.join([path, plugin])
+
+            if plugin.startswith('.'):
                 continue
             elif os.path.isdir(full_path):
                 continue
-            elif any(module.endswith(x) for x in C.BLACKLIST_EXTS):
+            elif any(plugin.endswith(x) for x in C.BLACKLIST_EXTS):
                 continue
-            elif module.startswith('__'):
+            elif plugin.startswith('__'):
                 continue
-            elif module in C.IGNORE_FILES:
+            elif plugin in C.IGNORE_FILES:
                 continue
-            elif module.startswith('_'):
+            elif plugin .startswith('_'):
                 if os.path.islink(full_path):  # avoids aliases
                     continue
 
-            module = os.path.splitext(module)[0]  # removes the extension
+            plugin = os.path.splitext(plugin)[0]  # removes the extension
-            module = module.lstrip('_')  # remove underscore from deprecated modules
+            plugin = plugin.lstrip('_')  # remove underscore from deprecated plugins
-            self.module_list.append(module)
 
-    def get_module_list_text(self):
+            if plugin not in plugin_docs.BLACKLIST.get(bkey, ()):
+                self.plugin_list.add(plugin)
+                display.vvvv("Added %s" % plugin)
+
+    def get_plugin_list_text(self, loader):
         columns = display.columns
-        displace = max(len(x) for x in self.module_list)
+        displace = max(len(x) for x in self.plugin_list)
         linelimit = columns - displace - 5
         text = []
         deprecated = []
-        for module in sorted(set(self.module_list)):
+        for plugin in sorted(self.plugin_list):
-
-            if module in module_docs.BLACKLIST_MODULES:
-                continue
 
             # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
-            filename = module_loader.find_plugin(module, mod_type='.py', ignore_deprecated=True)
+            filename = loader.find_plugin(plugin, mod_type='.py', ignore_deprecated=True)
 
             if filename is None:
                 continue
@@ -199,17 +220,23 @@ class DocCLI(CLI):
                 continue
 
             try:
-                doc, plainexamples, returndocs, metadata = module_docs.get_docstring(filename)
+                doc, plainexamples, returndocs, metadata = plugin_docs.get_docstring(filename)
-                desc = self.tty_ify(doc.get('short_description', '?')).strip()
-                if len(desc) > linelimit:
-                    desc = desc[:linelimit] + '...'
-
-                if module.startswith('_'): # Handle deprecated
-                    deprecated.append("%-*s %-*.*s" % (displace, module[1:], linelimit, len(desc), desc))
-                else:
-                    text.append("%-*s %-*.*s" % (displace, module, linelimit, len(desc), desc))
             except:
-                raise AnsibleError("module %s has a documentation error formatting or is missing documentation\n" % module)
+                display.warning("%s has a documentation formatting error" % plugin)
+
+            if not doc:
+                desc = 'UNDOCUMENTED'
+                display.warning("%s parsing did not produce documentation." % plugin)
+            else:
+                desc = self.tty_ify(doc.get('short_description', '?')).strip()
+
+            if len(desc) > linelimit:
+                desc = desc[:linelimit] + '...'
+
+            if plugin.startswith('_'): # Handle deprecated
+                deprecated.append("%-*s %-*.*s" % (displace, plugin[1:], linelimit, len(desc), desc))
+            else:
+                text.append("%-*s %-*.*s" % (displace, plugin, linelimit, len(desc), desc))
 
         if len(deprecated) > 0:
             text.append("\nDEPRECATED:")
@@ -253,34 +280,10 @@ class DocCLI(CLI):
 
         return "\n".join(text)
 
-    def get_man_text(self, doc):
+    def add_fields(self, text, fields, limit, opt_indent):
 
-        opt_indent="        "
+        for o in sorted(fields):
-        text = []
+            opt = fields[o]
-        text.append("> %s    (%s)\n" % (doc['module'].upper(), doc['filename']))
-        pad = display.columns * 0.20
-        limit = max(display.columns - int(pad), 70)
-
-        if isinstance(doc['description'], list):
-            desc = " ".join(doc['description'])
-        else:
-            desc = doc['description']
-
-        text.append("%s\n" % textwrap.fill(CLI.tty_ify(desc), limit, initial_indent="  ", subsequent_indent="  "))
-
-        # FUTURE: move deprecation to metadata-only
-
-        if 'deprecated' in doc and doc['deprecated'] is not None and len(doc['deprecated']) > 0:
-            text.append("DEPRECATED: \n%s\n" % doc['deprecated'])
-
-        if 'action' in doc and doc['action']:
-            text.append("  * note: %s\n" % "This module has a corresponding action plugin.")
-
-        if 'option_keys' in doc and len(doc['option_keys']) > 0:
-            text.append("Options (= is mandatory):\n")
-
-        for o in sorted(doc['option_keys']):
-            opt = doc['options'][o]
 
             required = opt.get('required', False)
             if not isinstance(required, bool):
@@ -306,6 +309,45 @@ class DocCLI(CLI):
                 default = "[Default: " +  str(opt.get('default', '(null)')) + "]"
             text.append(textwrap.fill(CLI.tty_ify(choices + default), limit, initial_indent=opt_indent, subsequent_indent=opt_indent))
 
+            for conf in ('config', 'env_vars', 'host_vars'):
+                if conf in opt:
+                    text.append(textwrap.fill(CLI.tty_ify("%s: " % conf), limit, initial_indent=opt_indent, subsequent_indent=opt_indent))
+                    for entry in opt[conf]:
+                        if isinstance(entry, dict):
+                            pre = "  -"
+                            for key in entry:
+                                text.append(textwrap.fill(CLI.tty_ify("%s %s: %s" % (pre, key, entry[key])),
+                                            limit, initial_indent=opt_indent, subsequent_indent=opt_indent))
+                                pre = "   "
+                        else:
+                            text.append(textwrap.fill(CLI.tty_ify("  - %s" % entry), limit, initial_indent=opt_indent, subsequent_indent=opt_indent))
+
+    def get_man_text(self, doc):
+
+        opt_indent="        "
+        text = []
+        text.append("> %s    (%s)\n" % (doc[self.options.type].upper(), doc['filename']))
+        pad = display.columns * 0.20
+        limit = max(display.columns - int(pad), 70)
+
+        if isinstance(doc['description'], list):
+            desc = " ".join(doc['description'])
+        else:
+            desc = doc['description']
+
+        text.append("%s\n" % textwrap.fill(CLI.tty_ify(desc), limit, initial_indent="  ", subsequent_indent="  "))
+
+        if 'deprecated' in doc and doc['deprecated'] is not None and len(doc['deprecated']) > 0:
+            text.append("DEPRECATED: \n%s\n" % doc['deprecated'])
+
+        if 'action' in doc and doc['action']:
+            text.append("  * note: %s\n" % "This module has a corresponding action plugin.")
+
+        if 'options' in doc and doc['options']:
+            text.append("Options (= is mandatory):\n")
+            self.add_fields(text, doc['options'], limit, opt_indent)
+            text.append('')
+
         if 'notes' in doc and doc['notes'] and len(doc['notes']) > 0:
             text.append("Notes:")
             for note in doc['notes']:
@@ -321,11 +363,18 @@ class DocCLI(CLI):
                 text.append("%s\n" % (ex['code']))
 
         if 'plainexamples' in doc and doc['plainexamples'] is not None:
-            text.append("EXAMPLES:")
+            text.append("EXAMPLES:\n")
-            text.append(doc['plainexamples'])
+            if isinstance(doc['plainexamples'], string_types):
+                text.append(doc['plainexamples'])
+            else:
+                text.append(yaml.dump(doc['plainexamples'], indent=2, default_flow_style=False))
+
         if 'returndocs' in doc and doc['returndocs'] is not None:
-            text.append("RETURN VALUES:")
+            text.append("RETURN VALUES:\n")
-            text.append(doc['returndocs'])
+            if isinstance(doc['returndocs'], string_types):
+                text.append(doc['returndocs'])
+            else:
+                text.append(yaml.dump(doc['returndocs'], indent=2, default_flow_style=False))
         text.append('')
 
         maintainers = set()
@@ -344,7 +393,7 @@ class DocCLI(CLI):
         text.append('MAINTAINERS: ' + ', '.join(maintainers))
         text.append('')
 
-        if doc['metadata'] and isinstance(doc['metadata'], dict):
+        if 'metadata' in doc and doc['metadata']:
             text.append("METADATA:")
             for k in doc['metadata']:
                 if isinstance(k, list):
@@ -352,5 +401,4 @@ class DocCLI(CLI):
                 else:
                     text.append("\t%s: %s" % (k.capitalize(), doc['metadata'][k]))
             text.append('')
-
         return "\n".join(text)

lib/ansible/plugins/cache/jsonfile.py

@@ -14,7 +14,15 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
+'''
+DOCUMENTATION:
+    cache: jsonfile
+    short_description: File backed, JSON formated.
+    description:
+        - File backed cache that uses JSON as a format, the files are per host.
+    version_added: "1.9"
+    author: Brian Coca (@bcoca)
+'''
 # Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type

lib/ansible/plugins/cache/memory.py

@@ -14,6 +14,17 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+'''
+DOCUMENTATION:
+    cache: memory
+    short_description: RAM backed, non persistent
+    description:
+        - RAM backed cache that is not persistent.
+    version_added: historical
+    author: core team (@ansible-core)
+'''
+
+
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 

lib/ansible/plugins/cache/pickle.py

@@ -14,6 +14,15 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+'''
+DOCUMENTATION:
+    cache: yaml
+    short_description: File backed, using Python's pickle.
+    description:
+        - File backed cache that uses Python's pickle serialization as a format, the files are per host.
+    version_added: "2.3"
+    author: Brian Coca (@bcoca)
+'''
 
 # Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)

lib/ansible/plugins/cache/yaml.py

@@ -14,6 +14,15 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+'''
+DOCUMENTATION:
+    cache: yaml
+    short_description: File backed, YAML formated.
+    description:
+        - File backed cache that uses YAML as a format, the files are per host.
+    version_added: "2.3"
+    author: Brian Coca (@bcoca)
+'''
 
 # Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)

lib/ansible/plugins/callback/default.py

@@ -14,7 +14,14 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
+'''
+DOCUMENTATION:
+    callback: default
+    short_description: default Ansbile screen output
+    version_added: historical
+    description:
+        - This is the default output callback for ansible-playbook.
+'''
 # Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type

lib/ansible/plugins/connection/local.py

@@ -15,6 +15,18 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+'''
+DOCUMENTATION:
+    connection: local
+    short_description: execute on controller
+    description:
+        - This connection plugin allows ansible to execute tasks on the Ansible 'controller' instead of on a remote host.
+    author: ansible (@core)
+    version_added: historical
+    notes:
+        - The remote user is ignored, the user with which the ansible CLI was executed is used instead.
+'''
+
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 

lib/ansible/plugins/connection/ssh.py

@@ -17,6 +17,84 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
+'''
+DOCUMENTATION:
+    connection: ssh
+    short_description: connect via ssh client binary
+    description:
+        - This connection plugin allows ansible to communicate to the target machines via normal ssh command line.
+    author: ansible (@core)
+    version_added: historical
+    options:
+        _host:
+            description: Hostname/ip to connect to.
+            default: inventory_hostname
+            host_vars:              
+                 - ansible_host
+                 - ansible_ssh_host
+        _host_key_checking:
+            type: bool
+            description: Determines if ssh should check host keys
+            config:
+                - section: defaults
+                  key: 'host_key_checking'
+            env_vars:
+                - ANSIBLE_HOST_KEY_CHECKING
+        _password:
+            description: Authentication password for the C(remote_user). Can be supplied as CLI option.
+            host_vars:
+                - ansible_password
+                - ansible_ssh_pass
+        _ssh_args:
+            description: Arguments to pass to all ssh cli tools
+            default: '-C -o ControlMaster=auto -o ControlPersist=60s'
+            config:
+                - section: 'ssh_connection'
+                  key: 'ssh_args'
+            env_vars:
+                - ANSIBLE_SSH_ARGS
+        _ssh_common_args:
+            description: Common extra args for ssh CLI tools
+            host_vars:
+                - ansible_ssh_common_args
+        _scp_extra_args:
+            description: Extra exclusive to the 'scp' CLI
+            host_vars:
+                - ansible_scp_extra_args
+        _sftp_extra_args:
+            description: Extra exclusive to the 'sftp' CLI
+            host_vars:
+                - ansible_sftp_extra_args
+        _ssh_extra_args:
+            description: Extra exclusive to the 'ssh' CLI
+            host_vars:
+                - ansible_ssh_extra_args
+        port:
+            description: Remote port to connect to.
+            type: int
+            config:
+               - section: defaults
+                 key: remote_port
+                 default: 22
+            env_vars:
+               - ANSIBLE_REMOTE_PORT
+            host_vars:
+               - ansible_port
+               - ansible_ssh_port
+        remote_user:
+            description:
+                - User name with which to login to the remote server, normally set by the remote_user keyword.
+                - If no user is supplied, Ansible will let the ssh client binary choose the user as it normally
+            config:
+               - section: defaults
+                 key: remote_user
+            env_vars:
+               - ANSIBLE_REMOTE_USER
+            host_vars:
+               - ansible_user
+               - ansible_ssh_user
+'''
+
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 

lib/ansible/plugins/lookup/cartesian.py

@@ -14,6 +14,34 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+"""
+DOCUMENTATION:
+    lookup: cartesian
+    version_added: "2.1"
+    short_description: returns the cartesian product of lists
+    description:
+        - Takes the input lists and returns a list that represents the product of the input lists.
+    options:
+        _raw:
+            description:
+                - a set of lists
+            required: True
+EXAMPLES:
+
+  - name: outputs the cartesian product of the supplied lists
+    debug: msg="{{item}}"
+    with_cartesian:
+         - "{{list1}}"
+         - "{{list2}}"
+  - name: used as lookup changes [1, 2, 3], [a, b] into [1, a], [1, b], [2, a], [2, b], [3, a], [3, b]
+    debug: msg="{{ [1,2,3]|lookup('cartesian', [a, b])}}"
+
+RETURN:
+  _list:
+    description:
+      - list of lists composed of elements of the input lists
+    type: lists
+"""
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 

lib/ansible/plugins/lookup/etcd.py

@@ -14,6 +14,43 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+'''
+DOCUMENTATION:
+    author:
+        - Jan-Piet Mens (@jpmens)
+    lookup: etcd
+    version_added: "2.1"
+    short_description: get info from etcd server
+    description:
+        - Retrieves data from an etcd server
+    options:
+        _raw:
+            description:
+                - the list of keys to lookup on the etcd server
+            type: string
+            required: True
+        _etcd_url:
+            description:
+                - Environment variable with the url for the etcd server
+            default: 'http://127.0.0.1:4001'
+            env_vars:
+                - ANSIBLE_ETCD_URL
+        _etcd_version:
+            description:
+                - Environment variable with the etcd protocol version
+            default: 'v1'
+            env_vars:
+                - ANSIBLE_ETCD_VERSION
+EXAMPLES:
+    - name: "a value from a locally running etcd"
+      debug: msg={{ lookup('etcd', 'foo') }}
+RETURN:
+    _list:
+        description:
+            - list of values associated with input keys
+        type: strings
+'''
+
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 

lib/ansible/plugins/strategy/debug.py

@@ -1,3 +1,26 @@
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+'''
+DOCUMENTATION:
+    strategy: debug
+    short_description: Executes tasks in interactive debug session.
+    description:
+        - Task execution is 'linear' but controlled by an interactive debug session.
+    version_added: "2.1"
+    author: Kishin Yagami
+'''
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type

lib/ansible/plugins/strategy/free.py

@@ -14,7 +14,17 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
+'''
+DOCUMENTATION:
+    strategy: free
+    short_description: Executes tasks on each host independently
+    description:
+        - Task execution is as fast as possible per host in batch as defined by C(serial) (default all).
+          Ansible will not wait for other hosts to finish the current task before queuing the next task for a host that has finished.
+          Once a host is done with the play, it opens it's slot to a new host that was waiting to start.
+    version_added: "2.0"
+    author: Ansible Core Team
+'''
 # Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type

lib/ansible/plugins/strategy/linear.py

@@ -14,7 +14,19 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
+'''
+DOCUMENTATION:
+    strategy: linear
+    short_description: Executes tasks in a linear fashion
+    description:
+        - Task execution is in lockstep per host batch as defined by C(serial) (default all).
+          Up to the fork limit of hosts will execute each task at the same time and then
+          the next series of hosts until the batch is done, before going on to the next task.
+    version_added: "2.0"
+    notes:
+     - This was the default Ansible behaviour before 'strategy plugins' were introduces in 2.0.
+    author: Ansible Core Team
+'''
 # Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type

lib/ansible/utils/module_docs.py

@@ -1,147 +0,0 @@
-# (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-# Make coding more python3-ish
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-import os
-import sys
-import ast
-import traceback
-
-from collections import MutableMapping, MutableSet, MutableSequence
-
-from ansible.compat.six import string_types
-from ansible.parsing.yaml.loader import AnsibleLoader
-from ansible.plugins import fragment_loader
-
-try:
-    from __main__ import display
-except ImportError:
-    from ansible.utils.display import Display
-    display = Display()
-
-# modules that are ok that they do not have documentation strings
-BLACKLIST_MODULES = frozenset((
-    'async_wrapper',
-))
-
-def get_docstring(filename, verbose=False):
-    """
-    Search for assignment of the DOCUMENTATION and EXAMPLES variables
-    in the given file.
-    Parse DOCUMENTATION from YAML and return the YAML doc or None
-    together with EXAMPLES, as plain text.
-
-    DOCUMENTATION can be extended using documentation fragments
-    loaded by the PluginLoader from the module_docs_fragments
-    directory.
-    """
-
-    doc = None
-    plainexamples = None
-    returndocs = None
-
-    # ensure metadata defaults
-    metadata = {'metadata_version': '1.0',
-                'status': ['preview'],
-                'supported_by': 'community'}
-
-    try:
-        # Thank you, Habbie, for this bit of code :-)
-        M = ast.parse(''.join(open(filename)))
-        for child in M.body:
-            if isinstance(child, ast.Assign):
-                for t in child.targets:
-                    try:
-                        theid = t.id
-                    except AttributeError as e:
-                        # skip errors can happen when trying to use the normal code
-                        display.warning("Failed to assign id for %s on %s, skipping" % (t, filename))
-                        continue
-
-                    if 'DOCUMENTATION' == theid:
-                        doc = AnsibleLoader(child.value.s, file_name=filename).get_single_data()
-                        fragments = doc.get('extends_documentation_fragment', [])
-
-                        if isinstance(fragments, string_types):
-                            fragments = [ fragments ]
-
-                        # Allow the module to specify a var other than DOCUMENTATION
-                        # to pull the fragment from, using dot notation as a separator
-                        for fragment_slug in fragments:
-                            fragment_slug = fragment_slug.lower()
-                            if '.' in fragment_slug:
-                                fragment_name, fragment_var = fragment_slug.split('.', 1)
-                                fragment_var = fragment_var.upper()
-                            else:
-                                fragment_name, fragment_var = fragment_slug, 'DOCUMENTATION'
-
-                            fragment_class = fragment_loader.get(fragment_name)
-                            assert fragment_class is not None
-
-                            fragment_yaml = getattr(fragment_class, fragment_var, '{}')
-                            fragment = AnsibleLoader(fragment_yaml, file_name=filename).get_single_data()
-
-                            if 'notes' in fragment:
-                                notes = fragment.pop('notes')
-                                if notes:
-                                    if 'notes' not in doc:
-                                        doc['notes'] = []
-                                    doc['notes'].extend(notes)
-
-                            if 'options' not in fragment:
-                                raise Exception("missing options in fragment (%s), possibly misformatted?: %s" % (fragment_name, filename))
-
-                            for key, value in fragment.items():
-                                if key not in doc:
-                                    doc[key] = value
-                                else:
-                                    if isinstance(doc[key], MutableMapping):
-                                        doc[key].update(value)
-                                    elif isinstance(doc[key], MutableSet):
-                                        doc[key].add(value)
-                                    elif isinstance(doc[key], MutableSequence):
-                                        doc[key] = sorted(frozenset(doc[key] + value))
-                                    else:
-                                        raise Exception("Attempt to extend a documentation fragement (%s) of unknown type: %s" % (fragment_name, filename))
-
-                    elif 'EXAMPLES' == theid:
-                        plainexamples = child.value.s[1:]  # Skip first empty line
-
-                    elif 'RETURN' == theid:
-                        returndocs = child.value.s[1:]
-
-                    elif 'ANSIBLE_METADATA' == theid:
-                        metadata = ast.literal_eval(child.value)
-                        if not isinstance(metadata, MutableMapping):
-                            # try yaml loading
-                            metadata = AnsibleLoader(metadata, file_name=filename).get_single_data()
-
-                        if not isinstance(metadata, MutableMapping):
-                            display.warning("Invalid metadata detected in %s, using defaults" % filename)
-                            metadata = {'status': ['preview'], 'supported_by': 'community', 'metadata_version': '1.0'}
-
-    except:
-        display.error("unable to parse %s" % filename)
-        if verbose is True:
-            display.display("unable to parse %s" % filename)
-            raise
-
-    return doc, plainexamples, returndocs, metadata

lib/ansible/utils/plugin_docs.py

@@ -0,0 +1,170 @@
+# (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import ast
+import yaml
+
+from collections import MutableMapping, MutableSet, MutableSequence
+
+from ansible.compat.six import string_types
+from ansible.parsing.yaml.loader import AnsibleLoader
+from ansible.plugins import fragment_loader
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+# modules that are ok that they do not have documentation strings
+BLACKLIST = {
+    'MODULE': frozenset(( 'async_wrapper',)),
+    'CACHE': frozenset(( 'base',)),
+}
+
+
+def add_fragments(doc, filename):
+
+    fragments = doc.get('extends_documentation_fragment', [])
+
+    if isinstance(fragments, string_types):
+        fragments = [ fragments ]
+
+    # Allow the module to specify a var other than DOCUMENTATION
+    # to pull the fragment from, using dot notation as a separator
+    for fragment_slug in fragments:
+        fragment_slug = fragment_slug.lower()
+        if '.' in fragment_slug:
+            fragment_name, fragment_var = fragment_slug.split('.', 1)
+            fragment_var = fragment_var.upper()
+        else:
+            fragment_name, fragment_var = fragment_slug, 'DOCUMENTATION'
+
+        fragment_class = fragment_loader.get(fragment_name)
+        assert fragment_class is not None
+
+        fragment_yaml = getattr(fragment_class, fragment_var, '{}')
+        fragment = AnsibleLoader(fragment_yaml, file_name=filename).get_single_data()
+
+        if 'notes' in fragment:
+            notes = fragment.pop('notes')
+            if notes:
+                if 'notes' not in doc:
+                    doc['notes'] = []
+                doc['notes'].extend(notes)
+
+        if 'options' not in fragment:
+            raise Exception("missing options in fragment (%s), possibly misformatted?: %s" % (fragment_name, filename))
+
+        for key, value in fragment.items():
+            if key not in doc:
+                doc[key] = value
+            else:
+                if isinstance(doc[key], MutableMapping):
+                    doc[key].update(value)
+                elif isinstance(doc[key], MutableSet):
+                    doc[key].add(value)
+                elif isinstance(doc[key], MutableSequence):
+                    doc[key] = sorted(frozenset(doc[key] + value))
+                else:
+                    raise Exception("Attempt to extend a documentation fragement (%s) of unknown type: %s" % (fragment_name, filename))
+
+
+def get_docstring(filename, verbose=False):
+    """
+    Search for assignment of the DOCUMENTATION and EXAMPLES variables
+    in the given file.
+    Parse DOCUMENTATION from YAML and return the YAML doc or None
+    together with EXAMPLES, as plain text.
+
+    DOCUMENTATION can be extended using documentation fragments
+    loaded by the PluginLoader from the module_docs_fragments
+    directory.
+    """
+
+    data = {
+        'doc': None,
+        'plainexamples': None,
+        'returndocs': None,
+        'metadata': None
+    }
+
+    string_to_vars = {
+        'DOCUMENTATION': 'doc',
+        'EXAMPLES': 'plainexamples',
+        'RETURN': 'returndocs',
+        'METADATA': 'metadata'
+    }
+
+    try:
+        M = ast.parse(''.join(open(filename)))
+        try:
+            display.debug('Attempt first docstring is yaml docs')
+            docstring = yaml.load(M.body[0].value.s)
+            for string in string_to_vars.keys():
+                if string in docstring:
+                    data[string_to_vars[string]] = docstring[string]
+                display.debug('assigned :%s' % string_to_vars[string])
+        except Exception as e:
+            display.debug('failed docstring parsing: %s' % str(e))
+
+        if not 'docs' in data or not data['docs']:
+            display.debug('Fallback to vars parsing')
+            for child in M.body:
+                if isinstance(child, ast.Assign):
+                    for t in child.targets:
+                        try:
+                            theid = t.id
+                        except AttributeError:
+                            # skip errors can happen when trying to use the normal code
+                            display.warning("Failed to assign id for %s on %s, skipping" % (t, filename))
+                            continue
+
+                        if theid in string_to_vars:
+                            varkey = string_to_vars[theid]
+                            if isinstance(child.value, MutableMapping):
+                                data[varkey] = child.value
+                            else:
+                                if theid in ['DOCUMENTATION', 'METADATA']:
+                                    # string should be yaml
+                                    data[varkey] = AnsibleLoader(child.value.s, file_name=filename).get_single_data()
+                                else:
+                                    # not yaml, should be a simple string
+                                    data[varkey] = child.value.s
+                            display.debug('assigned :%s' % varkey)
+
+        # add fragments to documentation
+        if data['doc']:
+            add_fragments(data['doc'], filename)
+
+        # remove version
+        if data['metadata']:
+            for x in ('version', 'metadata_version'):
+                if x in data['metadata']:
+                    del data['metadata'][x]
+    except:
+        display.error("unable to parse %s" % filename)
+        if verbose is True:
+            display.display("unable to parse %s" % filename)
+            raise
+
+    return data['doc'], data['plainexamples'], data['returndocs'], data['metadata']

test/sanity/validate-modules/validate-modules

@@ -37,7 +37,7 @@ from fnmatch import fnmatch
 
 from ansible import __version__ as ansible_version
 from ansible.executor.module_common import REPLACER_WINDOWS
-from ansible.utils.module_docs import BLACKLIST_MODULES, get_docstring
+from ansible.utils.plugin_docs import BLACKLIST, get_docstring
 
 from module_args import get_argument_spec
 
@@ -197,7 +197,7 @@ class ModuleValidator(Validator):
                                  'shippable.yml',
                                  '.gitattributes', '.gitmodules', 'COPYING',
                                  '__init__.py', 'VERSION', 'test-docs.sh'))
-    BLACKLIST = BLACKLIST_FILES.union(BLACKLIST_MODULES)
+    BLACKLIST = BLACKLIST_FILES.union(BLACKLIST['MODULE'])
 
     PS_DOC_BLACKLIST = frozenset((
         'async_status.ps1',