Update netconf_config module (#44379)

Fixes #40650
Fixes #40245
Fixes #41541

*  Refactor netconf_config module as per proposal #104
*  Update netconf_config module metadata to core network supported
*  Refactor local connection to use persistent connection framework
   for backward compatibility
*  Update netconf connection plugin configuration varaibles (Fixes #40245)
*  Add support for optional lock feature to Fixes #41541
*  Add integration test for netconf_config module
*  Documentation update
* Move deprecated options in netconf_config module

lib/ansible/plugins/action/netconf.py

@@ -0,0 +1,74 @@
+#
+# Copyright 2018 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import copy
+import sys
+
+from ansible.plugins.action.normal import ActionModule as _ActionModule
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+class ActionModule(_ActionModule):
+
+    def run(self, tmp=None, task_vars=None):
+        del tmp  # tmp no longer has any effect
+
+        if self._play_context.connection not in ['netconf', 'local'] and self._task.action == 'netconf_config':
+            return {'failed': True, 'msg': 'Connection type %s is not valid for netconf_config module. '
+                                           'Valid connection type is netconf or local (deprecated)' % self._play_context.connection}
+        elif self._play_context.connection not in ['netconf'] and self._task.action != 'netconf_config':
+            return {'failed': True, 'msg': 'Connection type %s is not valid for %s module. '
+                                           'Valid connection type is netconf.' % (self._play_context.connection, self._task.action)}
+
+        if self._play_context.connection == 'local' and self._task.action == 'netconf_config':
+            args = self._task.args
+            pc = copy.deepcopy(self._play_context)
+            pc.connection = 'netconf'
+            pc.port = int(args.get('port') or self._play_context.port or 830)
+
+            pc.remote_user = args.get('username') or self._play_context.connection_user
+            pc.password = args.get('password') or self._play_context.password
+            pc.private_key_file = args.get('ssh_keyfile') or self._play_context.private_key_file
+
+            display.vvv('using connection plugin %s (was local)' % pc.connection, pc.remote_addr)
+            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
+
+            timeout = args.get('timeout')
+            command_timeout = int(timeout) if timeout else connection.get_option('persistent_command_timeout')
+            connection.set_options(direct={'persistent_command_timeout': command_timeout, 'look_for_keys': args.get('look_for_keys'),
+                                           'hostkey_verify': args.get('hostkey_verify'),
+                                           'allow_agent': args.get('allow_agent')})
+
+            socket_path = connection.run()
+            display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
+            if not socket_path:
+                return {'failed': True,
+                        'msg': 'unable to open shell. Please see: ' +
+                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+
+            task_vars['ansible_socket'] = socket_path
+
+        return super(ActionModule, self).run(task_vars=task_vars)

lib/ansible/plugins/action/netconf_config.py

@@ -19,9 +19,94 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from ansible.plugins.action import ActionBase
-from ansible.plugins.action.net_config import ActionModule as NetActionModule
+import os
+import re
+import time
+import glob
+
+from ansible.plugins.action.netconf import ActionModule as _ActionModule
+from ansible.module_utils._text import to_text, to_bytes
+from ansible.module_utils.six.moves.urllib.parse import urlsplit
+
+PRIVATE_KEYS_RE = re.compile('__.+__')
 
 
-class ActionModule(NetActionModule, ActionBase):
-    pass
+class ActionModule(_ActionModule):
+
+    def run(self, tmp=None, task_vars=None):
+        if self._task.args.get('src'):
+            try:
+                self._handle_template()
+            except ValueError as exc:
+                return dict(failed=True, msg=to_text(exc))
+
+        result = super(ActionModule, self).run(tmp, task_vars)
+        del tmp  # tmp no longer has any effect
+
+        if self._task.args.get('backup') and result.get('__backup__'):
+            # User requested backup and no error occurred in module.
+            # NOTE: If there is a parameter error, _backup key may not be in results.
+            filepath = self._write_backup(task_vars['inventory_hostname'],
+                                          result['__backup__'])
+
+            result['backup_path'] = filepath
+
+        # strip out any keys that have two leading and two trailing
+        # underscore characters
+        for key in list(result):
+            if PRIVATE_KEYS_RE.match(key):
+                del result[key]
+
+        return result
+
+    def _get_working_path(self):
+        cwd = self._loader.get_basedir()
+        if self._task._role is not None:
+            cwd = self._task._role._role_path
+        return cwd
+
+    def _write_backup(self, host, contents):
+        backup_path = self._get_working_path() + '/backup'
+        if not os.path.exists(backup_path):
+            os.mkdir(backup_path)
+        for fn in glob.glob('%s/%s*' % (backup_path, host)):
+            os.remove(fn)
+        tstamp = time.strftime("%Y-%m-%d@%H:%M:%S", time.localtime(time.time()))
+        filename = '%s/%s_config.%s' % (backup_path, host, tstamp)
+        with open(filename, 'wb') as f:
+            f.write(to_bytes(to_text(contents, encoding='latin-1'), encoding='utf-8'))
+        return filename
+
+    def _handle_template(self):
+        src = self._task.args.get('src')
+        working_path = self._get_working_path()
+
+        if os.path.isabs(src) or urlsplit('src').scheme:
+            source = src
+        else:
+            source = self._loader.path_dwim_relative(working_path, 'templates', src)
+            if not source:
+                source = self._loader.path_dwim_relative(working_path, src)
+
+        if not os.path.exists(source):
+            raise ValueError('path specified in src not found')
+
+        try:
+            with open(source, 'r') as f:
+                template_data = to_text(f.read())
+        except IOError:
+            return dict(failed=True, msg='unable to load src file')
+
+        # Create a template search path in the following order:
+        # [working_path, self_role_path, dependent_role_paths, dirname(source)]
+        searchpath = [working_path]
+        if self._task._role is not None:
+            searchpath.append(self._task._role._role_path)
+            if hasattr(self._task, "_block:"):
+                dep_chain = self._task._block.get_dep_chain()
+                if dep_chain is not None:
+                    for role in dep_chain:
+                        searchpath.append(role._role_path)
+        searchpath.append(os.path.dirname(source))
+        self._templar.environment.loader.searchpath = searchpath
+        self._task.args['src'] = self._templar.template(template_data)

lib/ansible/plugins/connection/netconf.py

@@ -102,7 +102,8 @@ options:
       - name: ANSIBLE_HOST_KEY_AUTO_ADD
   look_for_keys:
     default: True
-    description: 'TODO: write it'
+    description:
+      -  enables looking for ssh keys in the usual locations for ssh keys (e.g. :file:`~/.ssh/id_*`)
     env:
       - name: ANSIBLE_PARAMIKO_LOOK_FOR_KEYS
     ini:
@@ -218,6 +219,7 @@ class Connection(NetworkConnectionBase):
         display.display('network_os is set to %s' % self._network_os, log_only=True)
 
         self._manager = None
+        self.key_filename = None
 
     def exec_command(self, cmd, in_data=None, sudoable=True):
         """Sends the request to the node and returns the reply
@@ -252,9 +254,9 @@ class Connection(NetworkConnectionBase):
             allow_agent = False
         setattr(self._play_context, 'allow_agent', allow_agent)
 
-        key_filename = None
-        if self._play_context.private_key_file:
-            key_filename = os.path.expanduser(self._play_context.private_key_file)
+        self.key_filename = self._play_context.private_key_file or self.get_option('private_key_file')
+        if self.key_filename:
+            self.key_filename = os.path.expanduser(self.key_filename)
 
         if self._network_os == 'default':
             for cls in netconf_loader.all(class_only=True):
@@ -277,7 +279,7 @@ class Connection(NetworkConnectionBase):
                 port=self._play_context.port or 830,
                 username=self._play_context.remote_user,
                 password=self._play_context.password,
-                key_filename=str(key_filename),
+                key_filename=str(self.key_filename),
                 hostkey_verify=self.get_option('host_key_checking'),
                 look_for_keys=self.get_option('look_for_keys'),
                 device_params=device_params,

lib/ansible/plugins/netconf/__init__.py

@@ -111,8 +111,6 @@ class NetconfBase(AnsiblePlugin):
         :param name: Name of rpc in string format
         :return: Received rpc response from remote host
         """
-        """RPC to be execute on remote device
-           :name: Name of rpc in string format"""
         try:
             obj = to_ele(name)
             resp = self.m.rpc(obj)
@@ -275,13 +273,19 @@ class NetconfBase(AnsiblePlugin):
         return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
 
     @ensure_connected
-    def locked(self, target):
+    def delete_config(self, target):
         """
-        Returns a context manager for a lock on a datastore
-        :param target: Name of the configuration datastore to lock
-        :return: Locked context object
+        delete a configuration datastore
+        :param target: specifies the  name or URL of configuration datastore to delete
+        :return: Returns xml string containing the RPC response received from remote host
         """
-        return self.m.locked(target)
+        resp = self.m.delete_config(target)
+        return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
+
+    @ensure_connected
+    def locked(self, *args, **kwargs):
+        resp = self.m.locked(*args, **kwargs)
+        return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
 
     @abstractmethod
     def get_capabilities(self):
@@ -341,6 +345,7 @@ class NetconfBase(AnsiblePlugin):
         operations['supports_startup'] = ':startup' in capabilities
         operations['supports_xpath'] = ':xpath' in capabilities
         operations['supports_writable_running'] = ':writable-running' in capabilities
+        operations['supports_validate'] = ':writable-validate' in capabilities
 
         operations['lock_datastore'] = []
         if operations['supports_writable_running']:

lib/ansible/plugins/netconf/ce.py

@@ -109,9 +109,9 @@ class Netconf(NetconfBase):
                 port=obj._play_context.port or 830,
                 username=obj._play_context.remote_user,
                 password=obj._play_context.password,
-                key_filename=obj._play_context.private_key_file,
-                hostkey_verify=C.HOST_KEY_CHECKING,
-                look_for_keys=C.PARAMIKO_LOOK_FOR_KEYS,
+                key_filename=obj.key_filename,
+                hostkey_verify=obj.get_option('host_key_checking'),
+                look_for_keys=obj.get_option('look_for_keys'),
                 allow_agent=obj._play_context.allow_agent,
                 timeout=obj._play_context.timeout
             )

lib/ansible/plugins/netconf/iosxr.py

@@ -104,9 +104,9 @@ class Netconf(NetconfBase):
                 port=obj._play_context.port or 830,
                 username=obj._play_context.remote_user,
                 password=obj._play_context.password,
-                key_filename=obj._play_context.private_key_file,
-                hostkey_verify=C.HOST_KEY_CHECKING,
-                look_for_keys=C.PARAMIKO_LOOK_FOR_KEYS,
+                key_filename=obj.key_filename,
+                hostkey_verify=obj.get_option('host_key_checking'),
+                look_for_keys=obj.get_option('look_for_keys'),
                 allow_agent=obj._play_context.allow_agent,
                 timeout=obj._play_context.timeout
             )

lib/ansible/plugins/netconf/junos.py

@@ -113,9 +113,9 @@ class Netconf(NetconfBase):
                 port=obj._play_context.port or 830,
                 username=obj._play_context.remote_user,
                 password=obj._play_context.password,
-                key_filename=obj._play_context.private_key_file,
-                hostkey_verify=C.HOST_KEY_CHECKING,
-                look_for_keys=C.PARAMIKO_LOOK_FOR_KEYS,
+                key_filename=obj.key_filename,
+                hostkey_verify=obj.get_option('host_key_checking'),
+                look_for_keys=obj.get_option('look_for_keys'),
                 allow_agent=obj._play_context.allow_agent,
                 timeout=obj._play_context.timeout
             )

lib/ansible/plugins/netconf/sros.py

@@ -82,9 +82,9 @@ class Netconf(NetconfBase):
                 port=obj._play_context.port or 830,
                 username=obj._play_context.remote_user,
                 password=obj._play_context.password,
-                key_filename=obj._play_context.private_key_file,
-                hostkey_verify=C.HOST_KEY_CHECKING,
-                look_for_keys=C.PARAMIKO_LOOK_FOR_KEYS,
+                key_filename=obj.key_filename,
+                hostkey_verify=obj.get_option('host_key_checking'),
+                look_for_keys=obj.get_option('look_for_keys'),
                 allow_agent=obj._play_context.allow_agent,
                 timeout=obj._play_context.timeout
             )