Connection plugins network_cli and netconf (#32521)

* implements jsonrpc message passing for ansible-connection

* implements more generic mechanism for persistent connections
* starts persistent connection in task_executor if enabled and supported
* supports using network_cli as top level connection plugin
* enhances logging for persistent connection to stdout

* Update action plugins

* Fix Python3 RPC

* Fix Junos bytes<-->str issues

* supports using netconf as top level connection plugin

* Error message when running netconf on an unsupported platform
* Update tests

* Fix `authorize: yes` for `connection: local`

* Handle potentially JSON data in terminal

* Add clarifying detail if possible on ConnectionError

lib/ansible/plugins/action/__init__.py

@@ -36,6 +36,7 @@ from ansible.module_utils.json_utils import _filter_non_json_lines
 from ansible.module_utils.six import binary_type, string_types, text_type, iteritems, with_metaclass
 from ansible.module_utils.six.moves import shlex_quote
 from ansible.module_utils._text import to_bytes, to_native, to_text
+from ansible.module_utils.connection import Connection
 from ansible.parsing.utils.jsonify import jsonify
 from ansible.release import __version__
 from ansible.utils.unsafe_proxy import wrap_var
@@ -604,7 +605,9 @@ class ActionBase(with_metaclass(ABCMeta, object)):
         module_args['_ansible_selinux_special_fs'] = C.DEFAULT_SELINUX_SPECIAL_FS
 
         # give the module the socket for persistent connections
-        module_args['_ansible_socket'] = task_vars.get('ansible_socket')
+        module_args['_ansible_socket'] = getattr(self._connection, 'socket_path')
+        if not module_args['_ansible_socket']:
+            module_args['_ansible_socket'] = task_vars.get('ansible_socket')
 
         # make sure all commands use the designated shell executable
         module_args['_ansible_shell_executable'] = self._play_context.executable
@@ -818,7 +821,8 @@ class ActionBase(with_metaclass(ABCMeta, object)):
         same_user = self._play_context.become_user == self._play_context.remote_user
         if sudoable and self._play_context.become and (allow_same_user or not same_user):
             display.debug("_low_level_execute_command(): using become for this command")
-            cmd = self._play_context.make_become_cmd(cmd, executable=executable)
+            if self._connection.transport != 'network_cli' and self._play_context.become_method != 'enable':
+                cmd = self._play_context.make_become_cmd(cmd, executable=executable)
 
         if self._connection.allow_executable:
             if executable is None:

lib/ansible/plugins/action/eos.py

@@ -40,47 +40,35 @@ class ActionModule(_ActionModule):
         provider = load_provider(eos_provider_spec, self._task.args)
         transport = provider['transport'] or 'cli'
 
-        if self._play_context.connection != 'local' and transport == 'cli':
-            return dict(
-                failed=True,
-                msg='invalid connection specified, expected connection=local, '
-                    'got %s' % self._play_context.connection
-            )
-
         display.vvvv('connection transport is %s' % transport, self._play_context.remote_addr)
 
         if transport == 'cli':
-            pc = copy.deepcopy(self._play_context)
-            pc.connection = 'network_cli'
-            pc.network_os = 'eos'
-            pc.remote_addr = provider['host'] or self._play_context.remote_addr
-            pc.port = int(provider['port'] or self._play_context.port or 22)
-            pc.remote_user = provider['username'] or self._play_context.connection_user
-            pc.password = provider['password'] or self._play_context.password
-            pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
-            pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
-            pc.become = provider['authorize'] or False
-            pc.become_pass = provider['auth_pass']
+            if self._play_context.connection == 'local':
+                pc = copy.deepcopy(self._play_context)
+                pc.connection = 'network_cli'
+                pc.network_os = 'eos'
+                pc.remote_addr = provider['host'] or self._play_context.remote_addr
+                pc.port = int(provider['port'] or self._play_context.port or 22)
+                pc.remote_user = provider['username'] or self._play_context.connection_user
+                pc.password = provider['password'] or self._play_context.password
+                pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
+                pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
+                pc.become = provider['authorize'] or False
+                if pc.become:
+                    pc.become_method = 'enable'
+                pc.become_pass = provider['auth_pass']
 
-            display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
-            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
+                display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
+                connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-            socket_path = connection.run()
-            display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
-            if not socket_path:
-                return {'failed': True,
-                        'msg': 'unable to open shell. Please see: ' +
-                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+                socket_path = connection.run()
+                display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
+                if not socket_path:
+                    return {'failed': True,
+                            'msg': 'unable to open shell. Please see: ' +
+                                   'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
-            # make sure we are in the right cli context which should be
-            # enable mode and not config module
-            rc, out, err = connection.exec_command('prompt()')
-            while '(config' in str(out):
-                display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
-                connection.exec_command('exit')
-                rc, out, err = connection.exec_command('prompt()')
-
-            task_vars['ansible_socket'] = socket_path
+                task_vars['ansible_socket'] = socket_path
 
         else:
             provider['transport'] = 'eapi'

lib/ansible/plugins/action/ios.py

@@ -38,50 +38,38 @@ class ActionModule(_ActionModule):
 
     def run(self, tmp=None, task_vars=None):
 
-        if self._play_context.connection != 'local':
-            return dict(
-                failed=True,
-                msg='invalid connection specified, expected connection=local, '
-                    'got %s' % self._play_context.connection
-            )
+        if self._play_context.connection == 'local':
+            provider = load_provider(ios_provider_spec, self._task.args)
 
-        provider = load_provider(ios_provider_spec, self._task.args)
+            pc = copy.deepcopy(self._play_context)
+            pc.connection = 'network_cli'
+            pc.network_os = 'ios'
+            pc.remote_addr = provider['host'] or self._play_context.remote_addr
+            pc.port = int(provider['port'] or self._play_context.port or 22)
+            pc.remote_user = provider['username'] or self._play_context.connection_user
+            pc.password = provider['password'] or self._play_context.password
+            pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
+            pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
+            pc.become = provider['authorize'] or False
+            if pc.become:
+                pc.become_method = 'enable'
+            pc.become_pass = provider['auth_pass']
 
-        pc = copy.deepcopy(self._play_context)
-        pc.connection = 'network_cli'
-        pc.network_os = 'ios'
-        pc.remote_addr = provider['host'] or self._play_context.remote_addr
-        pc.port = int(provider['port'] or self._play_context.port or 22)
-        pc.remote_user = provider['username'] or self._play_context.connection_user
-        pc.password = provider['password'] or self._play_context.password
-        pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
-        pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
-        pc.become = provider['authorize'] or False
-        pc.become_pass = provider['auth_pass']
+            display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
+            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-        display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
-        connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
+            socket_path = connection.run()
+            display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
+            if not socket_path:
+                return {'failed': True,
+                        'msg': 'unable to open shell. Please see: ' +
+                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
-        socket_path = connection.run()
-        display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
-        if not socket_path:
-            return {'failed': True,
-                    'msg': 'unable to open shell. Please see: ' +
-                           'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+            task_vars['ansible_socket'] = socket_path
 
-        # make sure we are in the right cli context which should be
-        # enable mode and not config module
-        rc, out, err = connection.exec_command('prompt()')
-        while str(out).strip().endswith(')#'):
-            display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
-            connection.exec_command('exit')
-            rc, out, err = connection.exec_command('prompt()')
-
-        task_vars['ansible_socket'] = socket_path
-
-        if self._play_context.become_method == 'enable':
-            self._play_context.become = False
-            self._play_context.become_method = None
+            if self._play_context.become_method == 'enable':
+                self._play_context.become = False
+                self._play_context.become_method = None
 
         result = super(ActionModule, self).run(tmp, task_vars)
         return result

lib/ansible/plugins/action/iosxr.py

@@ -38,43 +38,29 @@ class ActionModule(_ActionModule):
 
     def run(self, tmp=None, task_vars=None):
 
-        if self._play_context.connection != 'local':
-            return dict(
-                failed=True,
-                msg='invalid connection specified, expected connection=local, '
-                    'got %s' % self._play_context.connection
-            )
+        if self._play_context.connection == 'local':
+            provider = load_provider(iosxr_provider_spec, self._task.args)
 
-        provider = load_provider(iosxr_provider_spec, self._task.args)
+            pc = copy.deepcopy(self._play_context)
+            pc.connection = 'network_cli'
+            pc.network_os = 'iosxr'
+            pc.remote_addr = provider['host'] or self._play_context.remote_addr
+            pc.port = int(provider['port'] or self._play_context.port or 22)
+            pc.remote_user = provider['username'] or self._play_context.connection_user
+            pc.password = provider['password'] or self._play_context.password
+            pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
 
-        pc = copy.deepcopy(self._play_context)
-        pc.connection = 'network_cli'
-        pc.network_os = 'iosxr'
-        pc.remote_addr = provider['host'] or self._play_context.remote_addr
-        pc.port = int(provider['port'] or self._play_context.port or 22)
-        pc.remote_user = provider['username'] or self._play_context.connection_user
-        pc.password = provider['password'] or self._play_context.password
-        pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
+            display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
+            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-        display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
-        connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
+            socket_path = connection.run()
+            display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
+            if not socket_path:
+                return {'failed': True,
+                        'msg': 'unable to open shell. Please see: ' +
+                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
-        socket_path = connection.run()
-        display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
-        if not socket_path:
-            return {'failed': True,
-                    'msg': 'unable to open shell. Please see: ' +
-                           'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
-
-        # make sure we are in the right cli context which should be
-        # enable mode and not config module
-        rc, out, err = connection.exec_command('prompt()')
-        while str(out).strip().endswith(')#'):
-            display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
-            connection.exec_command('exit')
-            rc, out, err = connection.exec_command('prompt()')
-
-        task_vars['ansible_socket'] = socket_path
+            task_vars['ansible_socket'] = socket_path
 
         result = super(ActionModule, self).run(tmp, task_vars)
         return result

lib/ansible/plugins/action/junos.py

@@ -38,14 +38,6 @@ except ImportError:
 class ActionModule(_ActionModule):
 
     def run(self, tmp=None, task_vars=None):
-
-        if self._play_context.connection != 'local':
-            return dict(
-                failed=True,
-                msg='invalid connection specified, expected connection=local, '
-                    'got %s' % self._play_context.connection
-            )
-
         module = module_loader._load_module_source(self._task.action, module_loader.find_plugin(self._task.action))
 
         if not getattr(module, 'USE_PERSISTENT_CONNECTION', False):
@@ -72,25 +64,27 @@ class ActionModule(_ActionModule):
         pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
 
         display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
-        connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-        socket_path = connection.run()
-        display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
-        if not socket_path:
-            return {'failed': True,
-                    'msg': 'unable to open shell. Please see: ' +
-                           'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+        if self._play_context.connection == 'local':
+            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-        if pc.connection == 'network_cli':
-            # make sure we are in the right cli context which should be
-            # enable mode and not config module
-            rc, out, err = connection.exec_command('prompt()')
-            while str(out).strip().endswith(')#'):
-                display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
-                connection.exec_command('exit')
+            socket_path = connection.run()
+            display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
+            if not socket_path:
+                return {'failed': True,
+                        'msg': 'unable to open shell. Please see: ' +
+                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+
+            if pc.connection == 'network_cli':
+                # make sure we are in the right cli context which should be
+                # enable mode and not config module
                 rc, out, err = connection.exec_command('prompt()')
+                while str(out).strip().endswith(')#'):
+                    display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
+                    connection.exec_command('exit')
+                    rc, out, err = connection.exec_command('prompt()')
 
-        task_vars['ansible_socket'] = socket_path
+            task_vars['ansible_socket'] = socket_path
 
         result = super(ActionModule, self).run(tmp, task_vars)
         return result

lib/ansible/plugins/action/net_base.py

@@ -37,13 +37,6 @@ except ImportError:
 class ActionModule(ActionBase):
 
     def run(self, tmp=None, task_vars=None):
-        if self._play_context.connection != 'local':
-            return dict(
-                failed=True,
-                msg='invalid connection specified, expected connection=local, '
-                    'got %s' % self._play_context.connection
-            )
-
         play_context = copy.deepcopy(self._play_context)
         play_context.network_os = self._get_network_os(task_vars)
 
@@ -74,8 +67,9 @@ class ActionModule(ActionBase):
             play_context.become = self.provider['authorize'] or False
             play_context.become_pass = self.provider['auth_pass']
 
-        socket_path = self._start_connection(play_context)
-        task_vars['ansible_socket'] = socket_path
+        if self._play_context.connection == 'local':
+            socket_path = self._start_connection(play_context)
+            task_vars['ansible_socket'] = socket_path
 
         if 'fail_on_missing_module' not in self._task.args:
             self._task.args['fail_on_missing_module'] = False

lib/ansible/plugins/action/nxos.py

@@ -40,44 +40,31 @@ class ActionModule(_ActionModule):
         provider = load_provider(nxos_provider_spec, self._task.args)
         transport = provider['transport'] or 'cli'
 
-        if self._play_context.connection != 'local' and transport == 'cli':
-            return dict(
-                failed=True,
-                msg='invalid connection specified, expected connection=local, '
-                    'got %s' % self._play_context.connection
-            )
-
         display.vvvv('connection transport is %s' % transport, self._play_context.remote_addr)
 
         if transport == 'cli':
-            pc = copy.deepcopy(self._play_context)
-            pc.connection = 'network_cli'
-            pc.network_os = 'nxos'
-            pc.remote_addr = provider['host'] or self._play_context.remote_addr
-            pc.port = int(provider['port'] or self._play_context.port or 22)
-            pc.remote_user = provider['username'] or self._play_context.connection_user
-            pc.password = provider['password'] or self._play_context.password
-            pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
-            pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
-            display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
-            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
+            if self._play_context.connection == 'local':
+                pc = copy.deepcopy(self._play_context)
+                pc.connection = 'network_cli'
+                pc.network_os = 'nxos'
+                pc.remote_addr = provider['host'] or self._play_context.remote_addr
+                pc.port = int(provider['port'] or self._play_context.port or 22)
+                pc.remote_user = provider['username'] or self._play_context.connection_user
+                pc.password = provider['password'] or self._play_context.password
+                pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
+                pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
+                display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
 
-            socket_path = connection.run()
-            display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
-            if not socket_path:
-                return {'failed': True,
-                        'msg': 'unable to open shell. Please see: ' +
-                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+                connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-            # make sure we are in the right cli context which should be
-            # enable mode and not config module
-            rc, out, err = connection.exec_command('prompt()')
-            while str(out).strip().endswith(')#'):
-                display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
-                connection.exec_command('exit')
-                rc, out, err = connection.exec_command('prompt()')
+                socket_path = connection.run()
+                display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
+                if not socket_path:
+                    return {'failed': True,
+                            'msg': 'unable to open shell. Please see: ' +
+                                   'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
-            task_vars['ansible_socket'] = socket_path
+                task_vars['ansible_socket'] = socket_path
 
         else:
             provider['transport'] = 'nxapi'

lib/ansible/plugins/action/vyos.py

@@ -37,13 +37,6 @@ except ImportError:
 class ActionModule(_ActionModule):
 
     def run(self, tmp=None, task_vars=None):
-        if self._play_context.connection != 'local':
-            return dict(
-                failed=True,
-                msg='invalid connection specified, expected connection=local, '
-                    'got %s' % self._play_context.connection
-            )
-
         provider = load_provider(vyos_provider_spec, self._task.args)
 
         pc = copy.deepcopy(self._play_context)
@@ -57,24 +50,18 @@ class ActionModule(_ActionModule):
         pc.timeout = int(provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT)
 
         display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
-        connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-        socket_path = connection.run()
-        display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
-        if not socket_path:
-            return {'failed': True,
-                    'msg': 'unable to open shell. Please see: ' +
-                           'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
+        if self._play_context.connection == 'local':
+            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)
 
-        # make sure we are in the right cli context which should be
-        # enable mode and not config module
-        rc, out, err = connection.exec_command('prompt()')
-        while str(out).strip().endswith('#'):
-            display.vvvv('wrong context, sending exit to device', self._play_context.remote_addr)
-            connection.exec_command('exit')
-            rc, out, err = connection.exec_command('prompt()')
+            socket_path = connection.run()
+            display.vvvv('socket_path: %s' % socket_path, pc.remote_addr)
+            if not socket_path:
+                return {'failed': True,
+                        'msg': 'unable to open shell. Please see: ' +
+                               'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
-        task_vars['ansible_socket'] = socket_path
+            task_vars['ansible_socket'] = socket_path
 
         result = super(ActionModule, self).run(tmp, task_vars)
         return result