Connection plugins network_cli and netconf (#32521)

* implements jsonrpc message passing for ansible-connection

* implements more generic mechanism for persistent connections
* starts persistent connection in task_executor if enabled and supported
* supports using network_cli as top level connection plugin
* enhances logging for persistent connection to stdout

* Update action plugins

* Fix Python3 RPC

* Fix Junos bytes<-->str issues

* supports using netconf as top level connection plugin

* Error message when running netconf on an unsupported platform
* Update tests

* Fix `authorize: yes` for `connection: local`

* Handle potentially JSON data in terminal

* Add clarifying detail if possible on ConnectionError

lib/ansible/executor/task_executor.py

@@ -1,31 +1,21 @@
 # (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-# Make coding more python3-ish
+# (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import os
+import pty
 import time
+import json
+import subprocess
 import traceback
 
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleConnectionFailure, AnsibleActionFail, AnsibleActionSkip
 from ansible.executor.task_result import TaskResult
 from ansible.module_utils.six import iteritems, string_types, binary_type
+from ansible.module_utils.six.moves import cPickle
 from ansible.module_utils._text import to_text
 from ansible.playbook.conditional import Conditional
 from ansible.playbook.task import Task
@@ -490,6 +480,8 @@ class TaskExecutor:
                 not getattr(self._connection, 'connected', False) or
                 self._play_context.remote_addr != self._connection._play_context.remote_addr):
             self._connection = self._get_connection(variables=variables, templar=templar)
+            if getattr(self._connection, '_socket_path'):
+                variables['ansible_socket'] = self._connection._socket_path
             # only template the vars if the connection actually implements set_host_overrides
             # NB: this is expensive, and should be removed once connection-specific vars are being handled by play_context
             sho_impl = getattr(type(self._connection), 'set_host_overrides', None)
@@ -736,12 +728,7 @@ class TaskExecutor:
                     if isinstance(i, string_types) and i.startswith("ansible_") and i.endswith("_interpreter"):
                         variables[i] = delegated_vars[i]
 
-        # if using persistent paramiko connections (or the action has set the FORCE_PERSISTENT_CONNECTION attribute to True),
-        # then we use the persistent connection plugion. Otherwise load the requested connection plugin
-        if C.USE_PERSISTENT_CONNECTIONS or getattr(self, 'FORCE_PERSISTENT_CONNECTION', False):
-            conn_type = 'persistent'
-        else:
-            conn_type = self._play_context.connection
+        conn_type = self._play_context.connection
 
         connection = self._shared_loader_obj.connection_loader.get(conn_type, self._play_context, self._new_stdin)
         if not connection:
@@ -749,6 +736,13 @@ class TaskExecutor:
 
         self._play_context.set_options_from_plugin(connection)
 
+        if any(((connection.supports_persistence and C.USE_PERSISTENT_CONNECTIONS), connection.force_persistence)):
+            display.vvvv('attempting to start connection', host=self._play_context.remote_addr)
+            display.vvvv('using connection plugin %s' % connection.transport, host=self._play_context.remote_addr)
+            socket_path = self._start_connection()
+            display.vvvv('local domain socket path is %s' % socket_path, host=self._play_context.remote_addr)
+            setattr(connection, '_socket_path', socket_path)
+
         return connection
 
     def _get_action_handler(self, connection, templar):
@@ -780,3 +774,42 @@ class TaskExecutor:
             raise AnsibleError("the handler '%s' was not found" % handler_name)
 
         return handler
+
+    def _start_connection(self):
+        '''
+        Starts the persistent connection
+        '''
+        master, slave = pty.openpty()
+        p = subprocess.Popen(["ansible-connection"], stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdin = os.fdopen(master, 'wb', 0)
+        os.close(slave)
+
+        # Need to force a protocol that is compatible with both py2 and py3.
+        # That would be protocol=2 or less.
+        # Also need to force a protocol that excludes certain control chars as
+        # stdin in this case is a pty and control chars will cause problems.
+        # that means only protocol=0 will work.
+        src = cPickle.dumps(self._play_context.serialize(), protocol=0)
+        stdin.write(src)
+
+        stdin.write(b'\n#END_INIT#\n')
+
+        (stdout, stderr) = p.communicate()
+        stdin.close()
+
+        if p.returncode == 0:
+            result = json.loads(stdout)
+        else:
+            result = json.loads(stderr)
+
+        if 'messages' in result:
+            for msg in result.get('messages'):
+                display.vvvv('%s' % msg, host=self._play_context.remote_addr)
+
+        if 'error' in result:
+            if self._play_context.verbosity > 2:
+                msg = "The full traceback is:\n" + result['exception']
+                display.display(result['exception'], color=C.COLOR_ERROR)
+            raise AnsibleError(result['error'])
+
+        return result['socket_path']