Add support for cliconf and netconf plugin (#25093)

* ansible-connection refactor and action plugin changes
* Add cliconf plugin for eos, ios, iosxr, junos, nxos, vyos
* Add netconf plugin for junos
* Add jsonrpc support
* Modify network_cli and netconf connection plugin
* Fix py3 unit test failure
* Fix review comment
* Minor fixes
* Fix ansible-connection review comments
* Fix CI issue
* platform_agnostic related changes

lib/ansible/plugins/cliconf/__init__.py

@@ -0,0 +1,188 @@
+#
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import signal
+
+from abc import ABCMeta, abstractmethod
+from functools import wraps
+
+from ansible.errors import AnsibleError, AnsibleConnectionFailure
+from ansible.module_utils.six import with_metaclass
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+def enable_mode(func):
+    @wraps(func)
+    def wrapped(self, *args, **kwargs):
+        prompt = self.get_prompt()
+        if not str(prompt).strip().endswith('#'):
+            raise AnsibleError('operation requires privilege escalation')
+        return func(self, *args, **kwargs)
+    return wrapped
+
+
+class CliconfBase(with_metaclass(ABCMeta, object)):
+    """
+    A base class for implementing cli connections
+
+    .. note:: Unlike most of Ansible, nearly all strings in
+        :class:`CliconfBase` plugins are byte strings.  This is because of
+        how close to the underlying platform these plugins operate.  Remember
+        to mark literal strings as byte string (``b"string"``) and to use
+        :func:`~ansible.module_utils._text.to_bytes` and
+        :func:`~ansible.module_utils._text.to_text` to avoid unexpected
+        problems.
+
+    List of supported rpc's:
+        :get_config: Retrieves the specified configuration from the device
+        :edit_config: Loads the specified commands into the remote device
+        :get: Execute specified command on remote device
+        :get_capabilities: Retrieves device information and supported rpc methods
+        :commit: Load configuration from candidate to running
+        :discard_changes: Discard changes to candidate datastore
+
+    Note: List of supported rpc's for remote device can be extracted from
+          output of get_capabilities()
+
+    :returns: Returns output received from remote device as byte string
+
+            Usage:
+            from ansible.module_utils.connection import Connection
+
+            conn = Connection()
+            conn.get('show lldp neighbors detail'')
+            conn.get_config('running')
+            conn.edit_config(['hostname test', 'netconf ssh'])
+    """
+
+    def __init__(self, connection):
+        self._connection = connection
+
+    def _alarm_handler(self, signum, frame):
+        raise AnsibleConnectionFailure('timeout waiting for command to complete')
+
+    def send_command(self, command, prompt=None, answer=None, sendonly=False):
+        """Executes a cli command and returns the results
+        This method will execute the CLI command on the connection and return
+        the results to the caller.  The command output will be returned as a
+        string
+        """
+        timeout = self._connection._play_context.timeout or 30
+        signal.signal(signal.SIGALRM, self._alarm_handler)
+        signal.alarm(timeout)
+        display.display("command: %s" % command, log_only=True)
+        resp = self._connection.send(command, prompt, answer, sendonly)
+        signal.alarm(0)
+        return resp
+
+    def get_prompt(self):
+        """Returns the current prompt from the device"""
+        return self._connection._matched_prompt
+
+    def get_base_rpc(self):
+        """Returns list of base rpc method supported by remote device"""
+        return ['get_config', 'edit_config', 'get_capabilities', 'get']
+
+    @abstractmethod
+    def get_config(self, source='running', format='text'):
+        """Retrieves the specified configuration from the device
+        This method will retrieve the configuration specified by source and
+        return it to the caller as a string.  Subsequent calls to this method
+        will retrieve a new configuration from the device
+        :args:
+            arg[0] source: Datastore from which configuration should be retrieved eg: running/candidate/startup. (optional)
+                           default is running.
+            arg[1] format: Output format in which configuration is retrieved
+                           Note: Specified datastore should be supported by remote device.
+        :kwargs:
+          Keywords supported
+            :command: the command string to execute
+            :source: Datastore from which configuration should be retrieved
+            :format: Output format in which configuration is retrieved
+        :returns: Returns output received from remote device as byte string
+        """
+        pass
+
+    @abstractmethod
+    def edit_config(self, commands):
+        """Loads the specified commands into the remote device
+        This method will load the commands into the remote device.  This
+        method will make sure the device is in the proper context before
+        send the commands (eg config mode)
+        :args:
+            arg[0] command: List of configuration commands
+        :kwargs:
+          Keywords supported
+            :command: the command string to execute
+        :returns: Returns output received from remote device as byte string
+        """
+        pass
+
+    @abstractmethod
+    def get(self, *args, **kwargs):
+        """Execute specified command on remote device
+        This method will retrieve the specified data and
+        return it to the caller as a string.
+        :args:
+             arg[0] command: command in string format to be executed on remote device
+             arg[1] prompt: the expected prompt generated by executing command.
+                            This can be a string or a list of strings (optional)
+             arg[2] answer: the string to respond to the prompt with (optional)
+             arg[3] sendonly: bool to disable waiting for response, default is false (optional)
+        :kwargs:
+            :command: the command string to execute
+            :prompt: the expected prompt generated by executing command.
+                     This can be a string or a list of strings
+            :answer: the string to respond to the prompt with
+            :sendonly: bool to disable waiting for response
+        :returns: Returns output received from remote device as byte string
+        """
+        pass
+
+    @abstractmethod
+    def get_capabilities(self):
+        """Retrieves device information and supported
+        rpc methods by device platform and return result
+        as a string
+        :returns: Returns output received from remote device as byte string
+        """
+        pass
+
+    def commit(self, comment=None):
+        """Commit configuration changes"""
+        return self._connection.method_not_found("commit is not supported by network_os %s" % self._play_context.network_os)
+
+    def discard_changes(self):
+        "Discard changes in candidate datastore"
+        return self._connection.method_not_found("discard_changes is not supported by network_os %s" % self._play_context.network_os)
+
+    def put_file(self, source, destination):
+        """Copies file over scp to remote device"""
+        pass
+
+    def fetch_file(self, source, destination):
+        """Fetch file over scp from remote device"""
+        pass

lib/ansible/plugins/cliconf/eos.py

@@ -0,0 +1,73 @@
+#
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+from itertools import chain
+
+from ansible.module_utils.network_common import to_list
+from ansible.plugins.cliconf import CliconfBase, enable_mode
+
+
+class Cliconf(CliconfBase):
+
+    def get_device_info(self):
+        device_info = {}
+
+        device_info['network_os'] = 'eos'
+        reply = self.get(b'show version | json')
+        data = json.loads(reply)
+
+        device_info['network_os_version'] = data['version']
+        device_info['network_os_model'] = data['modelName']
+
+        reply = self.get(b'show hostname | json')
+        data = json.loads(reply)
+
+        device_info['network_os_hostname'] = data['hostname']
+
+        return device_info
+
+    @enable_mode
+    def get_config(self, source='running', format='text'):
+        lookup = {'running': 'running-config', 'startup': 'startup-config'}
+        if source not in lookup:
+            return self.invalid_params("fetching configuration from %s is not supported" % source)
+        if format == 'text':
+            cmd = b'show %s' % lookup[source]
+        else:
+            cmd = b'show %s | %s' % (lookup[source], format)
+        return self.send_command(cmd)
+
+    @enable_mode
+    def edit_config(self, command):
+        for cmd in chain([b'configure'], to_list(command), [b'end']):
+            self.send_command(cmd)
+
+    def get(self, *args, **kwargs):
+        return self.send_command(*args, **kwargs)
+
+    def get_capabilities(self):
+        result = {}
+        result['rpc'] = self.get_base_rpc()
+        result['network_api'] = 'cliconf'
+        result['device_info'] = self.get_device_info()
+        return json.dumps(result)

lib/ansible/plugins/cliconf/ios.py

@@ -0,0 +1,78 @@
+#
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import re
+import json
+
+from itertools import chain
+
+from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.network_common import to_list
+from ansible.plugins.cliconf import CliconfBase, enable_mode
+
+
+class Cliconf(CliconfBase):
+
+    def get_device_info(self):
+        device_info = {}
+
+        device_info['network_os'] = 'ios'
+        reply = self.get(b'show version')
+        data = to_text(reply, errors='surrogate_or_strict').strip()
+
+        match = re.search(r'Version (\S+),', data)
+        if match:
+            device_info['network_os_version'] = match.group(1)
+
+        match = re.search(r'^Cisco (.+) \(revision', data, re.M)
+        if match:
+            device_info['network_os_model'] = match.group(1)
+
+        match = re.search(r'^(.+) uptime', data, re.M)
+        if match:
+            device_info['network_os_hostname'] = match.group(1)
+
+        return device_info
+
+    @enable_mode
+    def get_config(self, source='running'):
+        if source not in ('running', 'startup'):
+            return self.invalid_params("fetching configuration from %s is not supported" % source)
+        if source == 'running':
+            cmd = b'show running-config all'
+        else:
+            cmd = b'show startup-config'
+        return self.send_command(cmd)
+
+    @enable_mode
+    def edit_config(self, command):
+        for cmd in chain([b'configure terminal'], to_list(command), [b'end']):
+            self.send_command(cmd)
+
+    def get(self, *args, **kwargs):
+        return self.send_command(*args, **kwargs)
+
+    def get_capabilities(self):
+        result = {}
+        result['rpc'] = self.get_base_rpc()
+        result['network_api'] = 'cliconf'
+        result['device_info'] = self.get_device_info()
+        return json.dumps(result)

lib/ansible/plugins/cliconf/iosxr.py

@@ -0,0 +1,87 @@
+#
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import re
+import json
+
+from itertools import chain
+
+from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.network_common import to_list
+from ansible.plugins.cliconf import CliconfBase
+
+
+class Cliconf(CliconfBase):
+
+    def get_device_info(self):
+        device_info = {}
+
+        device_info['network_os'] = 'iosxr'
+        reply = self.get(b'show version brief')
+        data = to_text(reply, errors='surrogate_or_strict').strip()
+
+        match = re.search(r'Version (\S+)$', data, re.M)
+        if match:
+            device_info['network_os_version'] = match.group(1)
+
+        match = re.search(r'image file is "(.+)"', data)
+        if match:
+            device_info['network_os_image'] = match.group(1)
+
+        match = re.search(r'^Cisco (.+) \(revision', data, re.M)
+        if match:
+            device_info['network_os_model'] = match.group(1)
+
+        match = re.search(r'^(.+) uptime', data, re.M)
+        if match:
+            device_info['network_os_hostname'] = match.group(1)
+
+        return device_info
+
+    def get_config(self, source='running'):
+        lookup = {'running': 'running-config'}
+        if source not in lookup:
+            return self.invalid_params("fetching configuration from %s is not supported" % source)
+        return self.send_command(to_bytes(b'show %s' % lookup[source], errors='surrogate_or_strict'))
+
+    def edit_config(self, command):
+        for cmd in chain([b'configure'], to_list(command), [b'end']):
+            self.send_command(cmd)
+
+    def get(self, *args, **kwargs):
+        return self.send_command(*args, **kwargs)
+
+    def commit(self, comment=None):
+        if comment:
+            command = b'commit comment {0}'.format(comment)
+        else:
+            command = b'commit'
+        self.send_command(command)
+
+    def discard_changes(self):
+        self.send_command(b'abort')
+
+    def get_capabilities(self):
+        result = {}
+        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes']
+        result['network_api'] = 'cliconf'
+        result['device_info'] = self.get_device_info()
+        return json.dumps(result)

lib/ansible/plugins/cliconf/junos.py

@@ -0,0 +1,87 @@
+#
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import re
+import json
+
+from itertools import chain
+from xml.etree.ElementTree import fromstring
+
+from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.network_common import to_list
+from ansible.plugins.cliconf import CliconfBase, enable_mode
+
+
+class Cliconf(CliconfBase):
+
+    def get_text(self, ele, tag):
+        try:
+            return to_text(ele.find(tag).text, errors='surrogate_then_replace').strip()
+        except AttributeError:
+            pass
+
+    def get_device_info(self):
+        device_info = {}
+
+        device_info['network_os'] = 'junos'
+        reply = self.get(b'show version | display xml')
+        data = fromstring(to_text(reply, errors='surrogate_then_replace').strip())
+
+        sw_info = data.find('.//software-information')
+
+        device_info['network_os_version'] = self.get_text(sw_info, 'junos-version')
+        device_info['network_os_hostname'] = self.get_text(sw_info, 'host-name')
+        device_info['network_os_model'] = self.get_text(sw_info, 'product-model')
+
+        return device_info
+
+    def get_config(self, source='running', format='text'):
+        if source != 'running':
+            return self.invalid_params("fetching configuration from %s is not supported" % source)
+        if format == 'text':
+            cmd = b'show configuration'
+        else:
+            cmd = b'show configuration | display %s' % format
+        return self.send_command(to_bytes(cmd), errors='surrogate_or_strict')
+
+    def edit_config(self, command):
+        for cmd in chain([b'configure'], to_list(command)):
+            self.send_command(cmd)
+
+    def get(self, *args, **kwargs):
+        return self.send_command(*args, **kwargs)
+
+    def commit(self, comment=None):
+        if comment:
+            command = b'commit comment {0}'.format(comment)
+        else:
+            command = b'commit'
+        self.send_command(command)
+
+    def discard_changes(self):
+        self.send_command(b'rollback')
+
+    def get_capabilities(self):
+        result = {}
+        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes']
+        result['network_api'] = 'cliconf'
+        result['device_info'] = self.get_device_info()
+        return json.dumps(result)

lib/ansible/plugins/cliconf/nxos.py

@@ -0,0 +1,62 @@
+#
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+from itertools import chain
+
+from ansible.module_utils.network_common import to_list
+from ansible.plugins.cliconf import CliconfBase
+
+
+class Cliconf(CliconfBase):
+
+    def get_device_info(self):
+        device_info = {}
+
+        device_info['network_os'] = 'nxos'
+        reply = self.get(b'show version | json')
+        data = json.loads(reply)
+
+        device_info['network_os_version'] = data['sys_ver_str']
+        device_info['network_os_model'] = data['chassis_id']
+        device_info['network_os_hostname'] = data['host_name']
+        device_info['network_os_image'] = data['isan_file_name']
+
+        return device_info
+
+    def get_config(self, source='running'):
+        lookup = {'running': 'running-config', 'startup': 'startup-config'}
+        return self.send_command(b'show %s' % lookup[source])
+
+    def edit_config(self, command):
+        for cmd in chain([b'configure'], to_list(command), [b'end']):
+            self.send_command(cmd)
+
+    def get(self, *args, **kwargs):
+        return self.send_command(*args, **kwargs)
+
+    def get_capabilities(self):
+        result = {}
+        result['rpc'] = self.get_base_rpc()
+        result['network_api'] = 'cliconf'
+        result['device_info'] = self.get_device_info()
+        return json.dumps(result)

lib/ansible/plugins/cliconf/vyos.py

@@ -0,0 +1,79 @@
+#
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import re
+import json
+
+from itertools import chain
+
+from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.network_common import to_list
+from ansible.plugins.cliconf import CliconfBase, enable_mode
+
+
+class Cliconf(CliconfBase):
+
+    def get_device_info(self):
+        device_info = {}
+
+        device_info['network_os'] = 'vyos'
+        reply = self.get(b'show version')
+        data = to_text(reply, errors='surrogate_or_strict').strip()
+
+        match = re.search(r'Version:\s*(\S+)', data)
+        if match:
+            device_info['network_os_version'] = match.group(1)
+
+        match = re.search(r'HW model:\s*(\S+)', data)
+        if match:
+            device_info['network_os_model'] = match.group(1)
+
+        reply = self.get(b'show host name')
+        device_info['network_os_hostname'] = to_text(reply, errors='surrogate_or_strict').strip()
+
+        return device_info
+
+    def get_config(self):
+        return self.send_command(b'show configuration all')
+
+    def edit_config(self, command):
+        for cmd in chain([b'configure'], to_list(command)):
+            self.send_command(cmd)
+
+    def get(self, *args, **kwargs):
+        return self.send_command(*args, **kwargs)
+
+    def commit(self, comment=None):
+        if comment:
+            command = b'commit comment {0}'.format(comment)
+        else:
+            command = b'commit'
+        self.send_command(command)
+
+    def discard_changes(self, *args, **kwargs):
+        self.send_command(b'discard')
+
+    def get_capabilities(self):
+        result = {}
+        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes']
+        result['network_api'] = 'cliconf'
+        result['device_info'] = self.get_device_info()
+        return json.dumps(result)