Deprecate Entity, EntityCollection and use subspec in network modules (#33575)

* Deprecate Entity, EntityCollection and use subspec in network modules

*  As per proposal https://github.com/ansible/proposals/issues/76
   deprecate use of Entity, EntityCollection, ComplexDict, ComplexList
   and use subspec instead.
*  Refactor ios modules
*  Refactor eos modules
*  Refactor vyos modules
*  Refactor nxos modules
*  Refactor iosxr modules
*  Add support for key in suboptions handling

* Fix CI issues

lib/ansible/modules/network/eos/eos_command.py

@@ -140,9 +140,8 @@ from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.pycompat24 import get_exception
 from ansible.module_utils.six import string_types
 from ansible.module_utils.network.common.parsing import Conditional
-from ansible.module_utils.network.common.utils import ComplexList
 from ansible.module_utils.network.eos.eos import run_commands
-from ansible.module_utils.network.eos.eos import eos_argument_spec, check_args
+from ansible.module_utils.network.eos.eos import eos_argument_spec
 
 VALID_KEYS = ['command', 'output', 'prompt', 'response']
 
@@ -157,16 +156,7 @@ def to_lines(stdout):
 
 
 def parse_commands(module, warnings):
-    spec = dict(
-        command=dict(key=True),
-        output=dict(),
-        prompt=dict(),
-        answer=dict()
-    )
-
-    transform = ComplexList(spec, module)
-    commands = transform(module.params['commands'])
-
+    commands = module.params['commands']
     if module.check_mode:
         for item in list(commands):
             if not item['command'].startswith('show'):
@@ -189,8 +179,15 @@ def to_cli(obj):
 def main():
     """entry point for module execution
     """
+    command_spec = dict(
+        command=dict(key=True),
+        output=dict(),
+        prompt=dict(),
+        answer=dict()
+    )
+
     argument_spec = dict(
-        commands=dict(type='list', required=True),
+        commands=dict(type='list', elements='dict', options=command_spec, required=True),
 
         wait_for=dict(type='list', aliases=['waitfor']),
         match=dict(default='all', choices=['all', 'any']),
@@ -207,7 +204,6 @@ def main():
     result = {'changed': False}
 
     warnings = list()
-    check_args(module, warnings)
     commands = parse_commands(module, warnings)
     if warnings:
         result['warnings'] = warnings

lib/ansible/modules/network/eos/eos_system.py

@@ -129,7 +129,6 @@ session_name:
 import re
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.network.common.utils import ComplexList
 from ansible.module_utils.network.eos.eos import load_config, get_config
 from ansible.module_utils.network.eos.eos import eos_argument_spec
 
@@ -270,31 +269,27 @@ def map_params_to_obj(module):
     obj = {
         'hostname': module.params['hostname'],
         'domain_name': module.params['domain_name'],
-        'domain_list': module.params['domain_list']
+        'domain_list': module.params['domain_list'],
+        'lookup_source': module.params['lookup_source'],
+        'name_servers': module.params['name_servers'],
     }
 
-    lookup_source = ComplexList(dict(
-        interface=dict(key=True),
-        vrf=dict()
-    ), module)
-
-    name_servers = ComplexList(dict(
-        server=dict(key=True),
-        vrf=dict(default='default')
-    ), module)
-
-    for arg, cast in [('lookup_source', lookup_source), ('name_servers', name_servers)]:
-        if module.params[arg] is not None:
-            obj[arg] = cast(module.params[arg])
-        else:
-            obj[arg] = None
-
     return obj
 
 
 def main():
     """ main entry point for module execution
     """
+    lookup_source_spec = dict(
+        interface=dict(key=True),
+        vrf=dict()
+    )
+
+    name_servers_spec = dict(
+        server=dict(key=True),
+        vrf=dict()
+    )
+
     argument_spec = dict(
         hostname=dict(),
 
@@ -302,10 +297,10 @@ def main():
         domain_list=dict(type='list', aliases=['domain_search']),
 
         # { interface: <str>, vrf: <str> }
-        lookup_source=dict(type='list'),
+        lookup_source=dict(type='list', elements='dict', options=lookup_source_spec),
 
         # { server: <str>; vrf: <str> }
-        name_servers=dict(type='list'),
+        name_servers=dict(type='list', elements='dict', options=name_servers_spec),
 
         state=dict(default='present', choices=['present', 'absent'])
     )
@@ -333,5 +328,6 @@ def main():
 
     module.exit_json(**result)
 
+
 if __name__ == '__main__':
     main()

lib/ansible/modules/network/ios/ios_command.py

@@ -134,9 +134,8 @@ failed_conditions:
 import time
 
 from ansible.module_utils.network.ios.ios import run_commands
-from ansible.module_utils.network.ios.ios import ios_argument_spec, check_args
+from ansible.module_utils.network.ios.ios import ios_argument_spec
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.network.common.utils import ComplexList
 from ansible.module_utils.network.common.parsing import Conditional
 from ansible.module_utils.six import string_types
 
@@ -149,13 +148,8 @@ def to_lines(stdout):
 
 
 def parse_commands(module, warnings):
-    command = ComplexList(dict(
-        command=dict(key=True),
-        prompt=dict(),
-        answer=dict()
-    ), module)
-    commands = command(module.params['commands'])
-    for item in list(commands):
+    commands = module.params['commands']
+    for item in commands:
         if module.check_mode and not item['command'].startswith('show'):
             warnings.append(
                 'only show commands are supported when using check mode, not '
@@ -173,8 +167,14 @@ def parse_commands(module, warnings):
 def main():
     """main entry point for module execution
     """
+    command_spec = dict(
+        command=dict(key=True),
+        prompt=dict(),
+        answer=dict()
+    )
+
     argument_spec = dict(
-        commands=dict(type='list', required=True),
+        commands=dict(type='list', elements='dict', options=command_spec, required=True),
 
         wait_for=dict(type='list', aliases=['waitfor']),
         match=dict(default='all', choices=['all', 'any']),
@@ -191,7 +191,6 @@ def main():
     result = {'changed': False}
 
     warnings = list()
-    check_args(module, warnings)
     commands = parse_commands(module, warnings)
     result['warnings'] = warnings
 

lib/ansible/modules/network/ios/ios_system.py

@@ -120,8 +120,7 @@ import re
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.ios.ios import get_config, load_config
-from ansible.module_utils.network.ios.ios import ios_argument_spec, check_args
-from ansible.module_utils.network.common.utils import ComplexList
+from ansible.module_utils.network.ios.ios import ios_argument_spec
 
 _CONFIGURED_VRFS = None
 
@@ -307,44 +306,43 @@ def map_params_to_obj(module):
         'hostname': module.params['hostname'],
         'lookup_source': module.params['lookup_source'],
         'lookup_enabled': module.params['lookup_enabled'],
+        'domain_name': module.params['domain_name'],
+        'domain_search': module.params['domain_search'],
+        'name_servers': module.params['name_servers']
     }
 
-    domain_name = ComplexList(dict(
-        name=dict(key=True),
-        vrf=dict()
-    ), module)
-
-    domain_search = ComplexList(dict(
-        name=dict(key=True),
-        vrf=dict()
-    ), module)
-
-    name_servers = ComplexList(dict(
-        server=dict(key=True),
-        vrf=dict()
-    ), module)
-
-    for arg, cast in [('domain_name', domain_name),
-                      ('domain_search', domain_search),
-                      ('name_servers', name_servers)]:
-
-        if module.params[arg]:
-            obj[arg] = cast(module.params[arg])
-        else:
-            obj[arg] = None
-
     return obj
 
 
 def main():
     """ Main entry point for Ansible module execution
     """
+    domain_name_spec = dict(
+        name=dict(key=True),
+        vrf=dict()
+    )
+
+    domain_search_spec = dict(
+        name=dict(key=True),
+        vrf=dict()
+    )
+
+    name_servers_spec = dict(
+        server=dict(key=True),
+        vrf=dict()
+    )
+
     argument_spec = dict(
         hostname=dict(),
 
-        domain_name=dict(type='list'),
-        domain_search=dict(type='list'),
-        name_servers=dict(type='list'),
+        # { name: <str>, vrf: <str> }
+        domain_name=dict(type='list', elements='dict', options=domain_name_spec),
+
+        # {name: <str>, vrf: <str> }
+        domain_search=dict(type='list', elements='dict', options=domain_search_spec),
+
+        # { server: <str>; vrf: <str> }
+        name_servers=dict(type='list', elements='dict', options=name_servers_spec),
 
         lookup_source=dict(),
         lookup_enabled=dict(type='bool'),
@@ -360,7 +358,6 @@ def main():
     result = {'changed': False}
 
     warnings = list()
-    check_args(module, warnings)
     result['warnings'] = warnings
 
     want = map_params_to_obj(module)
@@ -376,5 +373,6 @@ def main():
 
     module.exit_json(**result)
 
+
 if __name__ == "__main__":
     main()

lib/ansible/modules/network/iosxr/iosxr_command.py

@@ -127,8 +127,8 @@ import time
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.iosxr.iosxr import run_command, iosxr_argument_spec
-from ansible.module_utils.network.iosxr.iosxr import command_spec
 from ansible.module_utils.network.common.parsing import Conditional
+from ansible.module_utils.network.common.utils import to_list
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_native
 
@@ -142,11 +142,8 @@ def to_lines(stdout):
 
 def parse_commands(module, warnings):
     commands = module.params['commands']
-    for item in list(commands):
-        try:
-            command = item['command']
-        except Exception:
-            command = item
+    for item in to_list(commands):
+        command = item['command']
         if module.check_mode and not command.startswith('show'):
             warnings.append(
                 'only show commands are supported when using check mode, not '
@@ -163,8 +160,14 @@ def parse_commands(module, warnings):
 
 
 def main():
+    command_spec = dict(
+        command=dict(key=True),
+        prompt=dict(),
+        answer=dict()
+    )
+
     spec = dict(
-        commands=dict(type='list', required=True),
+        commands=dict(type='list', elements='dict', options=command_spec, required=True),
 
         wait_for=dict(type='list', aliases=['waitfor']),
         match=dict(default='all', choices=['all', 'any']),
@@ -175,13 +178,10 @@ def main():
 
     spec.update(iosxr_argument_spec)
 
-    spec.update(command_spec)
-
     module = AnsibleModule(argument_spec=spec,
                            supports_check_mode=True)
 
     warnings = list()
-
     commands = parse_commands(module, warnings)
 
     wait_for = module.params['wait_for'] or list()

lib/ansible/modules/network/nxos/nxos_command.py

@@ -147,8 +147,7 @@ import time
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.parsing import Conditional, FailedConditionalError
-from ansible.module_utils.network.common.utils import ComplexList
-from ansible.module_utils.network.nxos.nxos import check_args, nxos_argument_spec, run_commands
+from ansible.module_utils.network.nxos.nxos import nxos_argument_spec, run_commands
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_native
 
@@ -163,14 +162,7 @@ def to_lines(stdout):
 
 
 def parse_commands(module, warnings):
-    transform = ComplexList(dict(
-        command=dict(key=True),
-        output=dict(),
-        prompt=dict(),
-        answer=dict()
-    ), module)
-
-    commands = transform(module.params['commands'])
+    commands = module.params['commands']
 
     if module.check_mode:
         for item in list(commands):
@@ -194,9 +186,16 @@ def to_cli(obj):
 def main():
     """entry point for module execution
     """
+    command_spec = dict(
+        command=dict(key=True),
+        output=dict(),
+        prompt=dict(),
+        answer=dict()
+    )
+
     argument_spec = dict(
         # { command: <str>, output: <str>, prompt: <str>, response: <str> }
-        commands=dict(type='list', required=True),
+        commands=dict(type='list', elements='dict', options=command_spec, required=True),
 
         wait_for=dict(type='list', aliases=['waitfor']),
         match=dict(default='all', choices=['any', 'all']),
@@ -213,7 +212,6 @@ def main():
     result = {'changed': False}
 
     warnings = list()
-    check_args(module, warnings)
     commands = parse_commands(module, warnings)
     result['warnings'] = warnings
 

lib/ansible/modules/network/nxos/nxos_system.py

@@ -112,11 +112,10 @@ commands:
 import re
 
 from ansible.module_utils.network.nxos.nxos import get_config, load_config
-from ansible.module_utils.network.nxos.nxos import nxos_argument_spec, check_args
+from ansible.module_utils.network.nxos.nxos import nxos_argument_spec
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.six import iteritems
 from ansible.module_utils.network.common.config import NetworkConfig
-from ansible.module_utils.network.common.utils import ComplexList
 
 _CONFIGURED_VRFS = None
 
@@ -302,49 +301,45 @@ def map_params_to_obj(module):
     obj = {
         'hostname': module.params['hostname'],
         'domain_lookup': module.params['domain_lookup'],
-        'system_mtu': module.params['system_mtu']
+        'system_mtu': module.params['system_mtu'],
+        'domain_name': module.params['domain_name'],
+        'domain_search': module.params['domain_search'],
+        'name_servers': module.params['name_servers']
     }
 
-    domain_name = ComplexList(dict(
-        name=dict(key=True),
-        vrf=dict()
-    ), module)
-
-    domain_search = ComplexList(dict(
-        name=dict(key=True),
-        vrf=dict()
-    ), module)
-
-    name_servers = ComplexList(dict(
-        server=dict(key=True),
-        vrf=dict()
-    ), module)
-
-    for arg, cast in [('domain_name', domain_name), ('domain_search', domain_search),
-                      ('name_servers', name_servers)]:
-        if module.params[arg] is not None:
-            obj[arg] = cast(module.params[arg])
-        else:
-            obj[arg] = None
-
     return obj
 
 
 def main():
     """ main entry point for module execution
     """
+    domain_name_spec = dict(
+        name=dict(key=True),
+        vrf=dict()
+    )
+
+    domain_search_spec = dict(
+        name=dict(key=True),
+        vrf=dict()
+    )
+
+    name_servers_spec = dict(
+        server=dict(key=True),
+        vrf=dict()
+    )
+
     argument_spec = dict(
         hostname=dict(),
         domain_lookup=dict(type='bool'),
 
         # { name: <str>, vrf: <str> }
-        domain_name=dict(type='list'),
+        domain_name=dict(type='list', elements='dict', options=domain_name_spec),
 
         # {name: <str>, vrf: <str> }
-        domain_search=dict(type='list'),
+        domain_search=dict(type='list', elements='dict', options=domain_search_spec),
 
         # { server: <str>; vrf: <str> }
-        name_servers=dict(type='list'),
+        name_servers=dict(type='list', elements='dict', options=name_servers_spec),
 
         system_mtu=dict(type='int'),
         lookup_source=dict(),
@@ -357,7 +352,6 @@ def main():
                            supports_check_mode=True)
 
     warnings = list()
-    check_args(module, warnings)
 
     result = {'changed': False}
     if warnings:
@@ -376,5 +370,6 @@ def main():
 
     module.exit_json(**result)
 
+
 if __name__ == '__main__':
     main()

lib/ansible/modules/network/vyos/vyos_command.py

@@ -137,7 +137,6 @@ import time
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.pycompat24 import get_exception
 from ansible.module_utils.network.common.parsing import Conditional
-from ansible.module_utils.network.common.utils import ComplexList
 from ansible.module_utils.six import string_types
 from ansible.module_utils.network.vyos.vyos import run_commands
 from ansible.module_utils.network.vyos.vyos import vyos_argument_spec
@@ -151,12 +150,7 @@ def to_lines(stdout):
 
 
 def parse_commands(module, warnings):
-    command = ComplexList(dict(
-        command=dict(key=True),
-        prompt=dict(),
-        answer=dict(),
-    ), module)
-    commands = command(module.params['commands'])
+    commands = module.params['commands']
     items = []
 
     for item in commands:
@@ -170,8 +164,15 @@ def parse_commands(module, warnings):
 
 
 def main():
+
+    command_spec = dict(
+        command=dict(key=True),
+        prompt=dict(),
+        answer=dict()
+    )
+
     spec = dict(
-        commands=dict(type='list', required=True),
+        commands=dict(type='list', elements='dict', options=command_spec, required=True),
 
         wait_for=dict(type='list', aliases=['waitfor']),
         match=dict(default='all', choices=['all', 'any']),