network command modules cleanup (#45242)

* Standardize _command modules

* Centralize command transformation process

* Fix check mode error test

lib/ansible/module_utils/network/common/utils.py

@@ -34,6 +34,7 @@ from itertools import chain
 from struct import pack
 from socket import inet_aton, inet_ntoa
 
+from ansible.module_utils._text import to_text
 from ansible.module_utils.six import iteritems, string_types
 from ansible.module_utils.six.moves import zip
 from ansible.module_utils.basic import AnsibleFallbackNotFound
@@ -60,6 +61,26 @@ def to_list(val):
         return list()
 
 
+def to_lines(stdout):
+    for item in stdout:
+        if isinstance(item, string_types):
+            item = to_text(item).split('\n')
+        yield item
+
+
+def transform_commands(module):
+    transform = ComplexList(dict(
+        command=dict(key=True),
+        output=dict(),
+        prompt=dict(type='list'),
+        answer=dict(type='list'),
+        sendonly=dict(type='bool', default=False),
+        check_all=dict(type='bool', default=False),
+    ), module)
+
+    return transform(module.params['commands'])
+
+
 def sort_list(val):
     if isinstance(val, list):
         return sorted(val)

lib/ansible/modules/network/edgeos/edgeos_command.py

@@ -95,40 +95,25 @@ stdout_lines:
   type: list
   sample: [['...', '...'], ['...'], ['...']]
 """
-
 import time
 
 from ansible.module_utils._text import to_text
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.parsing import Conditional
-from ansible.module_utils.network.common.utils import ComplexList
+from ansible.module_utils.network.common.utils import transform_commands, to_lines
 from ansible.module_utils.network.edgeos.edgeos import run_commands
-from ansible.module_utils.six import string_types
-
-
-def to_lines(stdout):
-    for item in stdout:
-        if isinstance(item, string_types):
-            item = to_text(item).split('\n')
-        yield item
 
 
 def parse_commands(module, warnings):
-    spec = dict(
+    commands = transform_commands(module)
-        command=dict(key=True),
-        prompt=dict(),
-        answer=dict(),
-    )
-
-    transform = ComplexList(spec, module)
-    commands = transform(module.params['commands'])
 
     if module.check_mode:
         for item in list(commands):
             if not item['command'].startswith('show'):
                 warnings.append(
-                    'Only show commands are supported when using check_mode, '
+                    'Only show commands are supported when using check mode, not '
-                    'not executing %s' % item['command'])
+                    'executing %s' % item['command']
+                )
                 commands.remove(item)
 
     return commands
@@ -146,14 +131,14 @@ def main():
     module = AnsibleModule(argument_spec=spec, supports_check_mode=True)
 
     warnings = list()
-    result = {'changed': False}
+    result = {'changed': False, 'warnings': warnings}
     commands = parse_commands(module, warnings)
     wait_for = module.params['wait_for'] or list()
 
     try:
         conditionals = [Conditional(c) for c in wait_for]
-    except AttributeError as e:
+    except AttributeError as exc:
-        module.fail_json(msg=to_text(e))
+        module.fail_json(msg=to_text(exc))
 
     retries = module.params['retries']
     interval = module.params['interval']
@@ -180,12 +165,10 @@ def main():
         msg = 'One or more conditional statements have not been satisfied'
         module.fail_json(msg=msg, failed_conditions=failed_conditions)
 
-    result = {
+    result.update({
-        'changed': False,
         'stdout': responses,
-        'warnings': warnings,
+        'stdout_lines': list(to_lines(responses)),
-        'stdout_lines': list(to_lines(responses))
+    })
-    }
 
     module.exit_json(**result)
 

lib/ansible/modules/network/eos/eos_command.py

@@ -153,42 +153,24 @@ failed_conditions:
 """
 import time
 
+from ansible.module_utils._text import to_text
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils._text import to_native
-from ansible.module_utils.six import string_types
 from ansible.module_utils.network.common.parsing import Conditional
-from ansible.module_utils.network.common.utils import ComplexList
+from ansible.module_utils.network.common.utils import transform_commands, to_lines
 from ansible.module_utils.network.eos.eos import run_commands
 from ansible.module_utils.network.eos.eos import eos_argument_spec, check_args
 
 VALID_KEYS = ['command', 'output', 'prompt', 'response']
 
 
-def to_lines(stdout):
-    lines = list()
-    for item in stdout:
-        if isinstance(item, string_types):
-            item = str(item).split('\n')
-        lines.append(item)
-    return lines
-
-
 def parse_commands(module, warnings):
-    spec = dict(
+    commands = transform_commands(module)
-        command=dict(key=True),
-        output=dict(),
-        prompt=dict(),
-        answer=dict()
-    )
-
-    transform = ComplexList(spec, module)
-    commands = transform(module.params['commands'])
 
     if module.check_mode:
         for item in list(commands):
             if not item['command'].startswith('show'):
                 warnings.append(
-                    'Only show commands are supported when using check_mode, not '
+                    'Only show commands are supported when using check mode, not '
                     'executing %s' % item['command']
                 )
                 commands.remove(item)
@@ -221,20 +203,16 @@ def main():
     module = AnsibleModule(argument_spec=argument_spec,
                            supports_check_mode=True)
 
-    result = {'changed': False}
-
     warnings = list()
+    result = {'changed': False, 'warnings': warnings}
     check_args(module, warnings)
     commands = parse_commands(module, warnings)
-    if warnings:
-        result['warnings'] = warnings
-
     wait_for = module.params['wait_for'] or list()
 
     try:
         conditionals = [Conditional(c) for c in wait_for]
     except AttributeError as exc:
-        module.fail_json(msg=to_native(exc))
+        module.fail_json(msg=to_text(exc))
 
     retries = module.params['retries']
     interval = module.params['interval']
@@ -262,9 +240,8 @@ def main():
         module.fail_json(msg=msg, failed_conditions=failed_conditions)
 
     result.update({
-        'changed': False,
         'stdout': responses,
-        'stdout_lines': to_lines(responses)
+        'stdout_lines': list(to_lines(responses)),
     })
 
     module.exit_json(**result)

lib/ansible/modules/network/ios/ios_command.py

@@ -139,45 +139,28 @@ failed_conditions:
   type: list
   sample: ['...', '...']
 """
-import re
 import time
 
+from ansible.module_utils._text import to_text
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.network.common.parsing import Conditional
+from ansible.module_utils.network.common.utils import transform_commands, to_lines
 from ansible.module_utils.network.ios.ios import run_commands
 from ansible.module_utils.network.ios.ios import ios_argument_spec, check_args
-from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.network.common.utils import ComplexList
-from ansible.module_utils.network.common.parsing import Conditional
-from ansible.module_utils.six import string_types
-
-
-def to_lines(stdout):
-    for item in stdout:
-        if isinstance(item, string_types):
-            item = str(item).split('\n')
-        yield item
 
 
 def parse_commands(module, warnings):
-    command = ComplexList(dict(
+    commands = transform_commands(module)
-        command=dict(key=True),
+
-        prompt=dict(),
-        answer=dict()
-    ), module)
-    commands = command(module.params['commands'])
-    for item in list(commands):
-        configure_type = re.match(r'conf(?:\w*)(?:\s+(\w+))?', item['command'])
     if module.check_mode:
-            if configure_type and configure_type.group(1) not in ('confirm', 'replace', 'revert', 'network'):
+        for item in list(commands):
-                module.fail_json(
-                    msg='ios_command does not support running config mode '
-                        'commands.  Please use ios_config instead'
-                )
             if not item['command'].startswith('show'):
                 warnings.append(
-                    'only show commands are supported when using check mode, not '
+                    'Only show commands are supported when using check mode, not '
-                    'executing `%s`' % item['command']
+                    'executing %s' % item['command']
                 )
                 commands.remove(item)
+
     return commands
 
 
@@ -199,15 +182,16 @@ def main():
     module = AnsibleModule(argument_spec=argument_spec,
                            supports_check_mode=True)
 
-    result = {'changed': False}
-
     warnings = list()
+    result = {'changed': False, 'warnings': warnings}
     check_args(module, warnings)
     commands = parse_commands(module, warnings)
-    result['warnings'] = warnings
-
     wait_for = module.params['wait_for'] or list()
+
+    try:
         conditionals = [Conditional(c) for c in wait_for]
+    except AttributeError as exc:
+        module.fail_json(msg=to_text(exc))
 
     retries = module.params['retries']
     interval = module.params['interval']
@@ -235,9 +219,8 @@ def main():
         module.fail_json(msg=msg, failed_conditions=failed_conditions)
 
     result.update({
-        'changed': False,
         'stdout': responses,
-        'stdout_lines': list(to_lines(responses))
+        'stdout_lines': list(to_lines(responses)),
     })
 
     module.exit_json(**result)

lib/ansible/modules/network/iosxr/iosxr_command.py

@@ -121,19 +121,12 @@ failed_conditions:
 """
 import time
 
+from ansible.module_utils._text import to_text
 from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.network.common.parsing import Conditional
+from ansible.module_utils.network.common.utils import to_lines
 from ansible.module_utils.network.iosxr.iosxr import run_commands, iosxr_argument_spec
 from ansible.module_utils.network.iosxr.iosxr import command_spec
-from ansible.module_utils.network.common.parsing import Conditional
-from ansible.module_utils.six import string_types
-from ansible.module_utils._text import to_native
-
-
-def to_lines(stdout):
-    for item in stdout:
-        if isinstance(item, string_types):
-            item = to_native(item, errors='surrogate_or_strict').split('\n')
-        yield item
 
 
 def parse_commands(module, warnings):
@@ -145,21 +138,16 @@ def parse_commands(module, warnings):
             command = item
         if module.check_mode and not command.startswith('show'):
             warnings.append(
-                'only show commands are supported when using check mode, not '
+                'Only show commands are supported when using check mode, not '
-                'executing `%s`' % command
+                'executing %s' % command
             )
             commands.remove(item)
-        elif command.startswith('conf'):
-            module.fail_json(
-                msg='iosxr_command does not support running config mode '
-                    'commands.  Please use iosxr_config instead'
-            )
 
     return commands
 
 
 def main():
-    spec = dict(
+    argument_spec = dict(
         commands=dict(type='list', required=True),
 
         wait_for=dict(type='list', aliases=['waitfor']),
@@ -169,19 +157,21 @@ def main():
         interval=dict(default=1, type='int')
     )
 
-    spec.update(iosxr_argument_spec)
+    argument_spec.update(iosxr_argument_spec)
+    argument_spec.update(command_spec)
 
-    spec.update(command_spec)
+    module = AnsibleModule(argument_spec=argument_spec,
-
-    module = AnsibleModule(argument_spec=spec,
                            supports_check_mode=True)
 
     warnings = list()
-
+    result = {'changed': False, 'warnings': warnings}
     commands = parse_commands(module, warnings)
-
     wait_for = module.params['wait_for'] or list()
+
+    try:
         conditionals = [Conditional(c) for c in wait_for]
+    except AttributeError as exc:
+        module.fail_json(msg=to_text(exc))
 
     retries = module.params['retries']
     interval = module.params['interval']
@@ -208,12 +198,10 @@ def main():
         msg = 'One or more conditional statements have not been satisfied'
         module.fail_json(msg=msg, failed_conditions=failed_conditions)
 
-    result = {
+    result.update({
-        'changed': False,
         'stdout': responses,
-        'warnings': warnings,
+        'stdout_lines': list(to_lines(responses)),
-        'stdout_lines': list(to_lines(responses))
+    })
-    }
 
     module.exit_json(**result)
 

lib/ansible/modules/network/junos/junos_command.py

@@ -158,16 +158,17 @@ failed_conditions:
   type: list
   sample: ['...', '...']
 """
-import time
 import re
 import shlex
+import time
 
-from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_text
+from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.connection import ConnectionError
 from ansible.module_utils.network.common.netconf import exec_rpc
 from ansible.module_utils.network.junos.junos import junos_argument_spec, get_configuration, get_connection, get_capabilities, tostring
 from ansible.module_utils.network.common.parsing import Conditional, FailedConditionalError
+from ansible.module_utils.network.common.utils import to_lines
 from ansible.module_utils.six import string_types, iteritems
 
 
@@ -185,15 +186,6 @@ except ImportError:
 USE_PERSISTENT_CONNECTION = True
 
 
-def to_lines(stdout):
-    lines = list()
-    for item in stdout:
-        if isinstance(item, string_types):
-            item = str(item).split('\n')
-        lines.append(item)
-    return lines
-
-
 def rpc(module, items):
 
     responses = list()
@@ -438,7 +430,7 @@ def main():
         'changed': False,
         'warnings': warnings,
         'stdout': responses,
-        'stdout_lines': to_lines(responses)
+        'stdout_lines': list(to_lines(responses)),
     }
 
     if output:

lib/ansible/modules/network/nxos/nxos_command.py

@@ -124,38 +124,21 @@ failed_conditions:
 """
 import time
 
+from ansible.module_utils._text import to_text
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.parsing import Conditional, FailedConditionalError
-from ansible.module_utils.network.common.utils import ComplexList
+from ansible.module_utils.network.common.utils import transform_commands, to_lines
 from ansible.module_utils.network.nxos.nxos import check_args, nxos_argument_spec, run_commands
-from ansible.module_utils.six import string_types
-from ansible.module_utils._text import to_native
-
-
-def to_lines(stdout):
-    lines = list()
-    for item in stdout:
-        if isinstance(item, string_types):
-            item = str(item).split('\n')
-        lines.append(item)
-    return lines
 
 
 def parse_commands(module, warnings):
-    transform = ComplexList(dict(
+    commands = transform_commands(module)
-        command=dict(key=True),
-        output=dict(),
-        prompt=dict(),
-        answer=dict()
-    ), module)
-
-    commands = transform(module.params['commands'])
 
     if module.check_mode:
         for item in list(commands):
             if not item['command'].startswith('show'):
                 warnings.append(
-                    'Only show commands are supported when using check_mode, not '
+                    'Only show commands are supported when using check mode, not '
                     'executing %s' % item['command']
                 )
                 commands.remove(item)
@@ -189,19 +172,16 @@ def main():
     module = AnsibleModule(argument_spec=argument_spec,
                            supports_check_mode=True)
 
-    result = {'changed': False}
-
     warnings = list()
+    result = {'changed': False, 'warnings': warnings}
     check_args(module, warnings)
     commands = parse_commands(module, warnings)
-    result['warnings'] = warnings
-
     wait_for = module.params['wait_for'] or list()
 
     try:
         conditionals = [Conditional(c) for c in wait_for]
     except AttributeError as exc:
-        module.fail_json(msg=to_native(exc))
+        module.fail_json(msg=to_text(exc))
 
     retries = module.params['retries']
     interval = module.params['interval']
@@ -218,7 +198,7 @@ def main():
                         break
                     conditionals.remove(item)
             except FailedConditionalError as exc:
-                module.fail_json(msg=to_native(exc))
+                module.fail_json(msg=to_text(exc))
 
         if not conditionals:
             break
@@ -233,7 +213,7 @@ def main():
 
     result.update({
         'stdout': responses,
-        'stdout_lines': to_lines(responses)
+        'stdout_lines': list(to_lines(responses)),
     })
 
     module.exit_json(**result)

lib/ansible/modules/network/vyos/vyos_command.py

@@ -138,39 +138,27 @@ warnings:
 """
 import time
 
+from ansible.module_utils._text import to_text
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils._text import to_native
 from ansible.module_utils.network.common.parsing import Conditional
-from ansible.module_utils.network.common.utils import ComplexList
+from ansible.module_utils.network.common.utils import transform_commands, to_lines
-from ansible.module_utils.six import string_types
 from ansible.module_utils.network.vyos.vyos import run_commands
 from ansible.module_utils.network.vyos.vyos import vyos_argument_spec
 
 
-def to_lines(stdout):
-    for item in stdout:
-        if isinstance(item, string_types):
-            item = str(item).split('\n')
-        yield item
-
-
 def parse_commands(module, warnings):
-    command = ComplexList(dict(
+    commands = transform_commands(module)
-        command=dict(key=True),
-        prompt=dict(),
-        answer=dict(),
-    ), module)
-    commands = command(module.params['commands'])
-    items = []
 
-    for item in commands:
+    if module.check_mode:
-        if module.check_mode and not item['command'].startswith('show'):
+        for item in list(commands):
-            warnings.append('only show commands are supported when using '
+            if not item['command'].startswith('show'):
-                            'check mode, not executing `%s`' % item['command'])
+                warnings.append(
-        else:
+                    'Only show commands are supported when using check mode, not '
-            items.append(item)
+                    'executing %s' % item['command']
+                )
+                commands.remove(item)
 
-    return items
+    return commands
 
 
 def main():
@@ -189,14 +177,14 @@ def main():
     module = AnsibleModule(argument_spec=spec, supports_check_mode=True)
 
     warnings = list()
-
+    result = {'changed': False, 'warnings': warnings}
     commands = parse_commands(module, warnings)
-
     wait_for = module.params['wait_for'] or list()
+
     try:
         conditionals = [Conditional(c) for c in wait_for]
     except AttributeError as exc:
-        module.fail_json(msg=to_native(exc))
+        module.fail_json(msg=to_text(exc))
 
     retries = module.params['retries']
     interval = module.params['interval']
@@ -205,7 +193,7 @@ def main():
     for _ in range(retries):
         responses = run_commands(module, commands)
 
-        for item in conditionals:
+        for item in list(conditionals):
             if item(responses):
                 if match == 'any':
                     conditionals = list()
@@ -222,12 +210,10 @@ def main():
         msg = 'One or more conditional statements have not been satisfied'
         module.fail_json(msg=msg, failed_conditions=failed_conditions)
 
-    result = {
+    result.update({
-        'changed': False,
         'stdout': responses,
-        'warnings': warnings,
         'stdout_lines': list(to_lines(responses)),
-    }
+    })
 
     module.exit_json(**result)
 

test/units/modules/network/ios/fixtures/configure_revert_now

@@ -1,2 +0,0 @@
-%No Rollback Confirmed Change pending
-

test/units/modules/network/ios/fixtures/configure_terminal

@@ -0,0 +1,2 @@
+
+

test/units/modules/network/ios/test_ios_command.py

@@ -107,19 +107,20 @@ class TestIosCommandModule(TestIosModule):
         set_module_args(dict(commands=commands, wait_for=wait_for, match='all'))
         self.execute_module(failed=True)
 
-    def test_ios_command_configure_error(self):
+    def test_ios_command_configure_check_warning(self):
         commands = ['configure terminal']
         set_module_args({
             'commands': commands,
             '_ansible_check_mode': True,
         })
-        result = self.execute_module(failed=True)
+        result = self.execute_module()
         self.assertEqual(
-            result['msg'],
+            result['warnings'],
-            'ios_command does not support running config mode commands.  Please use ios_config instead'
+            ['Only show commands are supported when using check mode, not executing configure terminal'],
         )
 
-    def test_ios_command_configure_not_error(self):
+    def test_ios_command_configure_not_warning(self):
-        commands = ['configure revert now']
+        commands = ['configure terminal']
         set_module_args(dict(commands=commands))
-        self.execute_module()
+        result = self.execute_module()
+        self.assertEqual(result['warnings'], [])