Migrate command line parsing to argparse (#50610)

* Start of migration to argparse

* various fixes and improvements

* Linting fixes

* Test fixes

* Fix vault_password_files

* Add PrependAction for argparse

* A bunch of additional tweak/fixes

* Fix ansible-config tests

* Fix man page generation

* linting fix

* More adhoc pattern fixes

* Add changelog fragment

* Add support for argcomplete

* Enable argcomplete global completion

* Rename PrependAction to PrependListAction to better describe what it does

* Add documentation for installing and configuring argcomplete

* Address rebase issues

* Fix display encoding for vault

* Fix line length

* Address rebase issues

* Handle rebase issues

* Use mutually exclusive group instead of handling manually

* Fix rebase issues

* Address rebase issue

* Update version added for argcomplete support

* -e must be given a value

* ci_complete

docs/bin/generate_man.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import optparse
+import argparse
 import os
 import sys
 
@@ -11,15 +11,14 @@ from ansible.utils._build_helpers import update_file_if_different
 
 
 def generate_parser():
-    p = optparse.OptionParser(
-        version='%prog 1.0',
-        usage='usage: %prog [options]',
+    p = argparse.ArgumentParser(
         description='Generate cli documentation from cli docstrings',
     )
 
-    p.add_option("-t", "--template-file", action="store", dest="template_file", default="../templates/man.j2", help="path to jinja2 template")
-    p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
-    p.add_option("-f", "--output-format", action="store", dest="output_format", default='man', help="Output format for docs (the default 'man' or 'rst')")
+    p.add_argument("-t", "--template-file", action="store", dest="template_file", default="../templates/man.j2", help="path to jinja2 template")
+    p.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
+    p.add_argument("-f", "--output-format", action="store", dest="output_format", default='man', help="Output format for docs (the default 'man' or 'rst')")
+    p.add_argument('args', help='CLI module(s)', metavar='module', nargs='*')
     return p
 
 
@@ -57,34 +56,49 @@ def get_options(optlist):
     for opt in optlist:
         res = {
             'desc': opt.help,
-            'options': opt._short_opts + opt._long_opts
+            'options': opt.option_strings
         }
-        if opt.action == 'store':
+        if isinstance(opt, argparse._StoreAction):
             res['arg'] = opt.dest.upper()
+        elif not res['options']:
+            continue
         opts.append(res)
 
     return opts
 
 
+def dedupe_groups(parser):
+    action_groups = []
+    for action_group in parser._action_groups:
+        found = False
+        for a in action_groups:
+            if a._actions == action_group._actions:
+                found = True
+                break
+        if not found:
+            action_groups.append(action_group)
+    return action_groups
+
+
 def get_option_groups(option_parser):
     groups = []
-    for option_group in option_parser.option_groups:
+    for action_group in dedupe_groups(option_parser)[1:]:
         group_info = {}
-        group_info['desc'] = option_group.get_description()
-        group_info['options'] = option_group.option_list
-        group_info['group_obj'] = option_group
+        group_info['desc'] = action_group.description
+        group_info['options'] = action_group._actions
+        group_info['group_obj'] = action_group
         groups.append(group_info)
     return groups
 
 
-def opt_doc_list(cli):
+def opt_doc_list(parser):
     ''' iterate over options lists '''
 
     results = []
-    for option_group in cli.parser.option_groups:
-        results.extend(get_options(option_group.option_list))
+    for option_group in dedupe_groups(parser)[1:]:
+        results.extend(get_options(option_group._actions))
 
-    results.extend(get_options(cli.parser.option_list))
+    results.extend(get_options(parser._actions))
 
     return results
 
@@ -106,15 +120,17 @@ def opts_docs(cli_class_name, cli_module_name):
 
     # parse the common options
     try:
-        cli.parse()
+        cli.init_parser()
     except Exception:
         pass
 
+    cli.parser.prog = cli_name
+
     # base/common cli info
     docs = {
         'cli': cli_module_name,
         'cli_name': cli_name,
-        'usage': cli.parser.usage,
+        'usage': cli.parser.format_usage(),
         'short_desc': cli.parser.description,
         'long_desc': trim_docstring(cli.__doc__),
         'actions': {},
@@ -127,7 +143,7 @@ def opts_docs(cli_class_name, cli_module_name):
         if hasattr(cli, extras):
             docs[extras.lower()] = getattr(cli, extras)
 
-    common_opts = opt_doc_list(cli)
+    common_opts = opt_doc_list(cli.parser)
     groups_info = get_option_groups(cli.parser)
     shared_opt_names = []
     for opt in common_opts:
@@ -144,25 +160,11 @@ def opts_docs(cli_class_name, cli_module_name):
     # force populate parser with per action options
 
     # use class attrs not the attrs on a instance (not that it matters here...)
-    for action in getattr(cli_klass, 'VALID_ACTIONS', ()):
-        # instantiate each cli and ask its options
-        action_cli_klass = getattr(__import__("ansible.cli.%s" % cli_module_name,
-                                              fromlist=[cli_class_name]), cli_class_name)
-        # init with args with action added?
-        cli = action_cli_klass([])
-        cli.args.append(action)
-
-        try:
-            cli.parse()
-        except Exception:
-            pass
-
-        # FIXME/TODO: needed?
-        # avoid dupe errors
-        cli.parser.set_conflict_handler('resolve')
-
-        cli.set_action()
-
+    try:
+        subparser = cli.parser._subparsers._group_actions[0].choices
+    except AttributeError:
+        subparser = {}
+    for action, parser in subparser.items():
         action_info = {'option_names': [],
                        'options': []}
         # docs['actions'][action] = {}
@@ -171,7 +173,7 @@ def opts_docs(cli_class_name, cli_module_name):
         action_info['desc'] = trim_docstring(getattr(cli, 'execute_%s' % action).__doc__)
 
         # docs['actions'][action]['desc'] = getattr(cli, 'execute_%s' % action).__doc__.strip()
-        action_doc_list = opt_doc_list(cli)
+        action_doc_list = opt_doc_list(parser)
 
         uncommon_options = []
         for action_doc in action_doc_list:
@@ -196,7 +198,7 @@ def opts_docs(cli_class_name, cli_module_name):
 
         docs['actions'][action] = action_info
 
-    docs['options'] = opt_doc_list(cli)
+    docs['options'] = opt_doc_list(cli.parser)
     return docs
 
 
@@ -204,7 +206,7 @@ if __name__ == '__main__':
 
     parser = generate_parser()
 
-    options, args = parser.parse_args()
+    options = parser.parse_args()
 
     template_file = options.template_file
     template_path = os.path.expanduser(template_file)
@@ -214,7 +216,7 @@ if __name__ == '__main__':
     output_dir = os.path.abspath(options.output_dir)
     output_format = options.output_format
 
-    cli_modules = args
+    cli_modules = options.args
 
     # various cli parsing things checks sys.argv if the 'args' that are passed in are []
     # so just remove any args so the cli modules dont try to parse them resulting in warnings

docs/docsite/rst/installation_guide/intro_installation.rst

@@ -429,6 +429,91 @@ Now let's test things with a ping command:
 
 You can also use "sudo make install".
 
+.. _shell_completion:
+
+Shell Completion
+````````````````
+
+As of Ansible 2.9 shell completion of the ansible command line utilities is available and provided through an optional dependency
+called ``argcomplete``. ``argcomplete`` supports bash, and limited support for zsh and tcsh
+
+``python-argcomplete`` can be installed from EPEL on Red Hat Enterprise based distributions, and is available in the standard OS repositories for many other distributions.
+
+For more information about installing and configuration see the `argcomplete documentation <https://argcomplete.readthedocs.io/en/latest/>_`.
+
+Installing
+++++++++++
+
+via yum/dnf
+-----------
+
+On Fedora:
+
+.. code-block:: bash
+
+    $ sudo dnf install python-argcomplete
+
+On RHEL and CentOS:
+
+.. code-block:: bash
+
+    $ sudo yum install epel-release
+    $ sudo yum install python-argcomplete
+
+via apt
+-------
+
+.. code-block:: bash
+
+    $ sudo apt install python-argcomplete
+
+via pip
+-------
+
+.. code-block:: bash
+
+    $ pip install argcomplete
+
+Configuring
++++++++++++
+
+There are 2 ways to configure argcomplete to allow shell completion of the Ansible command line utilities. Per command, or globally.
+
+Globally
+--------
+
+Global completion requires bash 4.2
+
+.. code-block:: bash
+
+    $ sudo activate-global-python-argcomplete
+
+This will write a bash completion file to a global location, use ``--dest`` to change the location
+
+Per Command
+-----------
+
+If you do not have bash 4.2, you must register each script independently
+
+.. code-block:: bash
+
+    $ eval $(register-python-argcomplete ansible)
+    $ eval $(register-python-argcomplete ansible-config)
+    $ eval $(register-python-argcomplete ansible-console)
+    $ eval $(register-python-argcomplete ansible-doc)
+    $ eval $(register-python-argcomplete ansible-galaxy)
+    $ eval $(register-python-argcomplete ansible-inventory)
+    $ eval $(register-python-argcomplete ansible-playbook)
+    $ eval $(register-python-argcomplete ansible-pull)
+    $ eval $(register-python-argcomplete ansible-vault)
+
+It would be advisable to place the above commands, into your shells profile file such as ``~/.profile`` or ``~/.bash_profile``.
+
+Zsh or tcsh
+-----------
+
+See the `argcomplete documentation <https://argcomplete.readthedocs.io/en/latest/>_`.
+
 .. _getting_ansible:
 
 Ansible on GitHub

docs/docsite/rst/user_guide/playbooks_vault.rst

@@ -7,7 +7,7 @@ Using Vault in playbooks
 
 The "Vault" is a feature of Ansible that allows you to keep sensitive data such as passwords or keys in encrypted files, rather than as plaintext in playbooks or roles. These vault files can then be distributed or placed in source control.
 
-To enable this feature, a command line tool, :ref:`ansible-vault` is used to edit files, and a command line flag :option:`--ask-vault-pass <ansible-vault --ask-vault-pass>`, :option:`--vault-password-file <ansible-vault --vault-password-file>` or :option:`--vault-id <ansible-playbook --vault-id>` is used. You can also modify your ``ansible.cfg`` file to specify the location of a password file or configure Ansible to always prompt for the password. These options require no command line flag usage.
+To enable this feature, a command line tool, :ref:`ansible-vault` is used to edit files, and a command line flag :option:`--ask-vault-pass <ansible-vault-create --ask-vault-pass>`, :option:`--vault-password-file <ansible-vault-create --vault-password-file>` or :option:`--vault-id <ansible-playbook --vault-id>` is used. You can also modify your ``ansible.cfg`` file to specify the location of a password file or configure Ansible to always prompt for the password. These options require no command line flag usage.
 
 For best practices advice, refer to :ref:`best_practices_for_variables_and_vaults`.
 

docs/docsite/rst/user_guide/vault.rst

@@ -344,7 +344,7 @@ passwords will be tried in the order they are specified.
 In the above case, the 'dev' password will be tried first, then the 'prod' password for cases
 where Ansible doesn't know which vault ID is used to encrypt something.
 
-To add a vault ID label to the encrypted data use the :option:`--vault-id <ansible-vault --vault-id>` option
+To add a vault ID label to the encrypted data use the :option:`--vault-id <ansible-vault-create --vault-id>` option
 with a label when encrypting the data.
 
 The :ref:`DEFAULT_VAULT_ID_MATCH` config option can be set so that Ansible will only use the password with

docs/templates/cli_rst.j2

@@ -38,7 +38,7 @@ Common Options
 ==============
 
 
-{% for option in options|sort(attribute='options') %}
+{% for option in options|sort(attribute='options') if option.options %}
 
 .. option:: {% for switch in option['options'] %}{{switch}}{% if option['arg'] %} <{{option['arg']}}>{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}