Add junos_system declartive module and other related change (#25859)

* Add junos_system declartive module and other related change

*  junos_system declartive module
*  integration test for junos_system
*  integration test for net_system (junos platform)
*  pep8 fixes for junos modules
*  move to lxml from elementree for xml parsing as it support
   complete set of xpath api's
*  other minor changes

* Fix CI and doc changes

* Fix unit test failures

* Fix typo in import

* Fix import issue for py2.6

* Add missed Element in import

lib/ansible/modules/network/junos/_junos_template.py

@@ -21,7 +21,6 @@ ANSIBLE_METADATA = {'metadata_version': '1.0',
                     'supported_by': 'community'}
 
 
-
 DOCUMENTATION = """
 ---
 module: junos_template
@@ -87,6 +86,8 @@ options:
     required: false
     default: null
     choices: ['text', 'xml', 'set']
+requirements:
+  - ncclient (>=v0.5.2)
 notes:
   - This module requires the netconf system service be enabled on
     the remote device being managed
@@ -111,11 +112,11 @@ EXAMPLES = """
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.junos import check_args, junos_argument_spec
 from ansible.module_utils.junos import get_configuration, load_config
-from ansible.module_utils.six import text_type
 
 USE_PERSISTENT_CONNECTION = True
 DEFAULT_COMMENT = 'configured by junos_template'
 
+
 def main():
 
     argument_spec = dict(
@@ -137,16 +138,13 @@ def main():
 
     result = {'changed': False, 'warnings': warnings}
 
-    comment = module.params['comment']
-    confirm = module.params['confirm']
     commit = not module.check_mode
     action = module.params['action']
     src = module.params['src']
     fmt = module.params['config_format']
 
     if action == 'overwrite' and fmt == 'set':
-        module.fail_json(msg="overwrite cannot be used when format is "
-            "set per junos-pyez documentation")
+        module.fail_json(msg="overwrite cannot be used when format is set per junos-pyez documentation")
 
     if module.params['backup']:
         reply = get_configuration(module, format='set')

lib/ansible/modules/network/junos/junos_banner.py

@@ -54,6 +54,11 @@ options:
         present in the current devices active running configuration.
     default: present
     choices: ['present', 'absent', 'active', 'suspend']
+requirements:
+  - ncclient (>=v0.5.2)
+notes:
+  - This module requires the netconf system service be enabled on
+    the remote device being managed
 """
 
 EXAMPLES = """
@@ -102,12 +107,15 @@ rpc:
 """
 import collections
 
-from xml.etree.ElementTree import tostring
-
 from ansible.module_utils.junos import junos_argument_spec, check_args
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele
 
+try:
+    from lxml.etree import tostring
+except ImportError:
+    from xml.etree.ElementTree import tostring
+
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/modules/network/junos/junos_command.py

@@ -104,6 +104,10 @@ options:
     version_added: "2.3"
 requirements:
   - jxmlease
+  - ncclient (>=v0.5.2)
+notes:
+  - This module requires the netconf system service be enabled on
+    the remote device being managed
 """
 
 EXAMPLES = """
@@ -163,17 +167,17 @@ import time
 import re
 import shlex
 
-from functools import partial
-from xml.etree import ElementTree as etree
-from xml.etree.ElementTree import Element, SubElement, tostring
-
 from ansible.module_utils.junos import junos_argument_spec, check_args
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.netcli import Conditional, FailedConditionalError
 from ansible.module_utils.netconf import send_request
-from ansible.module_utils.network_common import ComplexList, to_list
 from ansible.module_utils.six import string_types, iteritems
 
+try:
+    from lxml.etree import Element, SubElement, tostring
+except ImportError:
+    from xml.etree.ElementTree import Element, SubElement, tostring
+
 try:
     import jxmlease
     HAS_JXMLEASE = True
@@ -182,6 +186,7 @@ except ImportError:
 
 USE_PERSISTENT_CONNECTION = True
 
+
 def to_lines(stdout):
     lines = list()
     for item in stdout:
@@ -190,6 +195,7 @@ def to_lines(stdout):
         lines.append(item)
     return lines
 
+
 def rpc(module, items):
 
     responses = list()
@@ -238,6 +244,7 @@ def rpc(module, items):
 
     return responses
 
+
 def split(value):
     lex = shlex.shlex(value)
     lex.quotes = '"'
@@ -245,6 +252,7 @@ def split(value):
     lex.commenters = ''
     return list(lex)
 
+
 def parse_rpcs(module):
     items = list()
 
@@ -270,6 +278,7 @@ def parse_rpcs(module):
 
     return items
 
+
 def parse_commands(module, warnings):
     items = list()
 
@@ -329,7 +338,6 @@ def main():
     items.extend(parse_rpcs(module))
 
     wait_for = module.params['wait_for'] or list()
-    display = module.params['display']
     conditionals = [Conditional(c) for c in wait_for]
 
     retries = module.params['retries']
@@ -344,8 +352,8 @@ def main():
         for item, resp in zip(items, responses):
             if item['xattrs']['format'] == 'xml':
                 if not HAS_JXMLEASE:
-                    module.fail_json(msg='jxmlease is required but does not appear to '
-                        'be installed.  It can be installed using `pip install jxmlease`')
+                    module.fail_json(msg='jxmlease is required but does not appear to be installed. '
+                                         'It can be installed using `pip install jxmlease`')
 
                 try:
                     transformed.append(jxmlease.parse(resp))
@@ -382,9 +390,7 @@ def main():
         'stdout_lines': to_lines(responses)
     }
 
-
     module.exit_json(**result)
 
-
 if __name__ == '__main__':
     main()

lib/ansible/modules/network/junos/junos_config.py

@@ -137,6 +137,8 @@ options:
     default: merge
     choices: ['merge', 'override', 'replace']
     version_added: "2.3"
+requirements:
+  - ncclient (>=v0.5.2)
 notes:
   - This module requires the netconf system service be enabled on
     the remote device being managed.
@@ -185,33 +187,47 @@ import re
 import json
 import sys
 
-from xml.etree import ElementTree
-
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.junos import get_diff, load_config, get_configuration
 from ansible.module_utils.junos import junos_argument_spec
 from ansible.module_utils.junos import check_args as junos_check_args
 from ansible.module_utils.netconf import send_request
 from ansible.module_utils.six import string_types
-from ansible.module_utils._text import to_text, to_native
+from ansible.module_utils._text import to_native
 
-if sys.version_info < (2, 7):
-    from xml.parsers.expat import ExpatError
-    ParseError = ExpatError
-else:
-    ParseError = ElementTree.ParseError
+try:
+    from lxml.etree import Element, fromstring
+except ImportError:
+    from xml.etree.ElementTree import Element, fromstring
+
+try:
+    from lxml.etree import ParseError
+except ImportError:
+    try:
+        from xml.etree.ElementTree import ParseError
+    except ImportError:
+        # for Python < 2.7
+        from xml.parsers.expat import ExpatError
+        ParseError = ExpatError
 
 USE_PERSISTENT_CONNECTION = True
 DEFAULT_COMMENT = 'configured by junos_config'
 
+
 def check_args(module, warnings):
     junos_check_args(module, warnings)
 
     if module.params['replace'] is not None:
         module.fail_json(msg='argument replace is deprecated, use update')
 
-zeroize = lambda x: send_request(x, ElementTree.Element('request-system-zeroize'))
-rollback = lambda x: get_diff(x)
+
+def zeroize(ele):
+    return send_request(ele, Element('request-system-zeroize'))
+
+
+def rollback(ele):
+    return get_diff(ele)
+
 
 def guess_format(config):
     try:
@@ -221,7 +237,7 @@ def guess_format(config):
         pass
 
     try:
-        ElementTree.fromstring(config)
+        fromstring(config)
         return 'xml'
     except ParseError:
         pass
@@ -231,6 +247,7 @@ def guess_format(config):
 
     return 'text'
 
+
 def filter_delete_statements(module, candidate):
     reply = get_configuration(module, format='set')
     match = reply.find('.//configuration-set')
@@ -248,6 +265,7 @@ def filter_delete_statements(module, candidate):
 
     return modified_candidate
 
+
 def configure_device(module, warnings):
     candidate = module.params['lines'] or module.params['src']
 
@@ -283,6 +301,7 @@ def configure_device(module, warnings):
 
     return load_config(module, candidate, warnings, **kwargs)
 
+
 def main():
     """ main entry point for module execution
     """

lib/ansible/modules/network/junos/junos_facts.py

@@ -59,9 +59,13 @@ options:
     default: text
     choices: ['xml', 'set', 'text', 'json']
     version_added: "2.3"
+requirements:
+  - ncclient (>=v0.5.2)
 notes:
   - Ensure I(config_format) used to retrieve configuration from device
     is supported by junos version running on device.
+  - This module requires the netconf system service be enabled on
+    the remote device being managed
 """
 
 EXAMPLES = """
@@ -79,16 +83,18 @@ ansible_facts:
   returned: always
   type: dict
 """
-
-from xml.etree.ElementTree import Element, SubElement, tostring
-
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.pycompat24 import get_exception
 from ansible.module_utils.six import iteritems
 from ansible.module_utils.junos import junos_argument_spec, check_args, get_param
-from ansible.module_utils.junos import command, get_configuration
+from ansible.module_utils.junos import get_configuration
 from ansible.module_utils.netconf import send_request
 
+try:
+    from lxml.etree import Element, SubElement, tostring
+except ImportError:
+    from xml.etree.ElementTree import Element, SubElement, tostring
+
 try:
     from jnpr.junos import Device
     from jnpr.junos.exception import ConnectError

lib/ansible/modules/network/junos/junos_interface.py

@@ -76,6 +76,11 @@ options:
       - State of the Interface configuration.
     default: present
     choices: ['present', 'absent', 'active', 'suspend']
+requirements:
+  - ncclient (>=v0.5.2)
+notes:
+  - This module requires the netconf system service be enabled on
+    the remote device being managed
 """
 
 EXAMPLES = """
@@ -136,12 +141,15 @@ rpc:
 """
 import collections
 
-from xml.etree.ElementTree import tostring
-
 from ansible.module_utils.junos import junos_argument_spec, check_args
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele
 
+try:
+    from lxml.etree import tostring
+except ImportError:
+    from xml.etree.ElementTree import tostring
+
 USE_PERSISTENT_CONNECTION = True
 
 
@@ -193,7 +201,7 @@ def main():
 
     param_to_xpath_map = collections.OrderedDict()
     param_to_xpath_map.update({
-        'name': 'name',
+        'name': {'xpath': 'name', 'is_key': True},
         'description': 'description',
         'speed': 'speed',
         'mtu': 'mtu',

lib/ansible/modules/network/junos/junos_netconf.py

@@ -105,11 +105,13 @@ def map_obj_to_commands(updates, module):
 
     return commands
 
+
 def parse_port(config):
     match = re.search(r'port (\d+)', config)
     if match:
         return int(match.group(1))
 
+
 def map_config_to_obj(module):
     cmd = 'show configuration system services netconf'
     rc, out, err = exec_command(module, cmd)
@@ -130,6 +132,7 @@ def validate_netconf_port(value, module):
     if not 1 <= value <= 65535:
         module.fail_json(msg='netconf_port must be between 1 and 65535')
 
+
 def map_params_to_obj(module):
     obj = {
         'netconf_port': module.params['netconf_port'],
@@ -144,6 +147,7 @@ def map_params_to_obj(module):
 
     return obj
 
+
 def load_config(module, config, commit=False):
 
     exec_command(module, 'configure')
@@ -164,6 +168,7 @@ def load_config(module, config, commit=False):
 
     return str(diff).strip()
 
+
 def main():
     """main entry point for module execution
     """

lib/ansible/modules/network/junos/junos_package.py

@@ -79,6 +79,7 @@ options:
     choices: ['true', 'false']
 requirements:
   - junos-eznc
+  - ncclient (>=v0.5.2)
 notes:
   - This module requires the netconf system service be enabled on
     the remote device being managed
@@ -142,7 +143,8 @@ def install_package(module, device):
     package = module.params['src']
     no_copy = module.params['no_copy']
 
-    progress_log = lambda x, y: module.log(y)
+    def progress_log(dev, report):
+        module.log(report)
 
     module.log('installing package')
     result = junos.install(package, progress=progress_log, no_copy=no_copy)

lib/ansible/modules/network/junos/junos_rpc.py

@@ -55,6 +55,11 @@ options:
         version of software that supports native JSON output.
     required: false
     default: xml
+requirements:
+  - ncclient (>=v0.5.2)
+notes:
+  - This module requires the netconf system service be enabled on
+    the remote device being managed
 """
 
 EXAMPLES = """
@@ -84,8 +89,6 @@ output_lines:
   returned: always
   type: list
 """
-from xml.etree.ElementTree import Element, SubElement, tostring
-
 from ansible.module_utils.junos import junos_argument_spec, check_args
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.netconf import send_request
@@ -93,6 +96,11 @@ from ansible.module_utils.six import iteritems
 
 USE_PERSISTENT_CONNECTION = True
 
+try:
+    from lxml.etree import Element, SubElement, tostring
+except ImportError:
+    from xml.etree.ElementTree import Element, SubElement, tostring
+
 
 def main():
     """main entry point for Ansible module

lib/ansible/modules/network/junos/junos_system.py

@@ -0,0 +1,195 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2017, Ansible by Red Hat, inc
+#
+# This file is part of Ansible by Red Hat
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+ANSIBLE_METADATA = {'metadata_version': '1.0',
+                    'status': ['preview'],
+                    'supported_by': 'core'}
+
+
+DOCUMENTATION = """
+---
+module: junos_system
+version_added: "2.4"
+author: "Ganesh Nalawade (@ganeshrn)"
+short_description: Manage the system attributes on Juniper JUNOS devices
+description:
+  - This module provides declarative management of node system attributes
+    on Juniper JUNOS devices.  It provides an option to configure host system
+    parameters or remove those parameters from the device active
+    configuration.
+options:
+  hostname:
+    description:
+      - Configure the device hostname parameter. This option takes an ASCII string value.
+  domain_name:
+    description:
+      - Configure the IP domain name
+        on the remote device to the provided value. Value
+        should be in the dotted name form and will be
+        appended to the C(hostname) to create a fully-qualified
+        domain name.
+  domain_search:
+    description:
+      - Provides the list of domain suffixes to
+        append to the hostname for the purpose of doing name resolution.
+        This argument accepts a list of names and will be reconciled
+        with the current active configuration on the running node.
+  name_servers:
+    description:
+      - List of DNS name servers by IP address to use to perform name resolution
+        lookups.  This argument accepts either a list of DNS servers See
+        examples.
+  state:
+    description:
+      - State of the configuration
+        values in the device's current active configuration.  When set
+        to I(present), the values should be configured in the device active
+        configuration and when set to I(absent) the values should not be
+        in the device active configuration
+    default: present
+    choices: ['present', 'absent', 'active', 'suspend']
+requirements:
+  - ncclient (>=v0.5.2)
+notes:
+  - This module requires the netconf system service be enabled on
+    the remote device being managed
+"""
+
+EXAMPLES = """
+- name: configure hostname and domain name
+  junos_system:
+    hostname: junos01
+    domain_name: test.example.com
+    domain-search:
+      - ansible.com
+      - redhat.com
+      - juniper.com
+
+- name: remove configuration
+  junos_system:
+    state: absent
+
+- name: configure name servers
+  junos_system:
+    name_servers:
+      - 8.8.8.8
+      - 8.8.4.4
+"""
+
+RETURN = """
+rpc:
+  description: load-configuration RPC send to the device
+  returned: when configuration is changed on device
+  type: string
+  sample: >
+            <interfaces>
+                <interface>
+                    <name>ge-0/0/0</name>
+                    <description>test interface</description>
+                </interface>
+            </interfaces>
+"""
+import collections
+
+from ansible.module_utils.junos import junos_argument_spec, check_args
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele
+
+try:
+    from lxml.etree import tostring
+except ImportError:
+    from xml.etree.ElementTree import tostring
+
+USE_PERSISTENT_CONNECTION = True
+
+
+def validate_param_values(module, obj):
+    for key in obj:
+        # validate the param value (if validator func exists)
+        validator = globals().get('validate_%s' % key)
+        if callable(validator):
+            validator(module.params.get(key), module)
+
+
+def main():
+    """ main entry point for module execution
+    """
+    argument_spec = dict(
+        hostname=dict(),
+        domain_name=dict(),
+        domain_search=dict(type='list'),
+        name_servers=dict(type='list'),
+
+        state=dict(choices=['present', 'absent', 'active', 'suspend'], default='present')
+    )
+
+    argument_spec.update(junos_argument_spec)
+
+    params = ['hostname', 'domain_name', 'domain_search', 'name_servers']
+    required_if = [('state', 'present', params, True),
+                   ('state', 'absent', params, True),
+                   ('state', 'active', params, True),
+                   ('state', 'suspend', params, True)]
+
+    module = AnsibleModule(argument_spec=argument_spec,
+                           required_if=required_if,
+                           supports_check_mode=True)
+
+    warnings = list()
+    check_args(module, warnings)
+
+    result = {'changed': False}
+
+    if warnings:
+        result['warnings'] = warnings
+
+    top = 'system'
+
+    param_to_xpath_map = collections.OrderedDict()
+    param_to_xpath_map.update({
+        'hostname': {'xpath': 'host-name', 'leaf_only': True},
+        'domain_name': {'xpath': 'domain-name', 'leaf_only': True},
+        'domain_search': {'xpath': 'domain-search', 'leaf_only': True, 'value_req': True},
+        'name_servers': {'xpath': 'name-server/name', 'is_key': True}
+    })
+
+    validate_param_values(module, param_to_xpath_map)
+
+    want = list()
+    want.append(map_params_to_obj(module, param_to_xpath_map))
+    ele = map_obj_to_ele(module, want, top)
+
+    kwargs = {'commit': not module.check_mode}
+    kwargs['action'] = 'replace'
+
+    diff = load_config(module, tostring(ele), warnings, **kwargs)
+
+    if diff:
+        result.update({
+            'changed': True,
+            'diff': {'prepared': diff},
+            'rpc': tostring(ele)
+        })
+
+    module.exit_json(**result)
+
+if __name__ == "__main__":
+    main()

lib/ansible/modules/network/junos/junos_user.py

@@ -91,6 +91,11 @@ options:
     required: false
     default: present
     choices: ['present', 'absent']
+requirements:
+  - ncclient (>=v0.5.2)
+notes:
+  - This module requires the netconf system service be enabled on
+    the remote device being managed
 """
 
 EXAMPLES = """
@@ -116,13 +121,16 @@ RETURN = """
 """
 from functools import partial
 
-from xml.etree.ElementTree import Element, SubElement, tostring
-
 from ansible.module_utils.junos import junos_argument_spec, check_args
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.junos import load_config
 from ansible.module_utils.six import iteritems
 
+try:
+    from lxml.etree import Element, SubElement, tostring
+except ImportError:
+    from xml.etree.ElementTree import Element, SubElement, tostring
+
 ROLES = ['operator', 'read-only', 'super-user', 'unauthorized']
 USE_PERSISTENT_CONNECTION = True
 

lib/ansible/modules/network/junos/junos_vlan.py

@@ -60,6 +60,11 @@ options:
       - State of the VLAN configuration.
     default: present
     choices: ['present', 'absent', 'active', 'suspend']
+requirements:
+  - ncclient (>=v0.5.2)
+notes:
+  - This module requires the netconf system service be enabled on
+    the remote device being managed
 """
 
 EXAMPLES = """
@@ -94,12 +99,15 @@ rpc:
 """
 import collections
 
-from xml.etree.ElementTree import tostring
-
 from ansible.module_utils.junos import junos_argument_spec, check_args
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.junos import load_config, map_params_to_obj, map_obj_to_ele
 
+try:
+    from lxml.etree import tostring
+except ImportError:
+    from xml.etree.ElementTree import tostring
+
 USE_PERSISTENT_CONNECTION = True
 
 
@@ -147,7 +155,7 @@ def main():
 
     param_to_xpath_map = collections.OrderedDict()
     param_to_xpath_map.update({
-        'name': 'name',
+        'name': {'xpath': 'name', 'is_key': True},
         'vlan_id': 'vlan-id',
         'description': 'description'
     })