Connection plugins network_cli and netconf (#32521)

* implements jsonrpc message passing for ansible-connection

* implements more generic mechanism for persistent connections
* starts persistent connection in task_executor if enabled and supported
* supports using network_cli as top level connection plugin
* enhances logging for persistent connection to stdout

* Update action plugins

* Fix Python3 RPC

* Fix Junos bytes<-->str issues

* supports using netconf as top level connection plugin

* Error message when running netconf on an unsupported platform
* Update tests

* Fix `authorize: yes` for `connection: local`

* Handle potentially JSON data in terminal

* Add clarifying detail if possible on ConnectionError

lib/ansible/plugins/connection/persistent.py

@@ -1,4 +1,4 @@
-# (c) 2017 Red Hat Inc.
+# 2017 Red Hat Inc.
 # (c) 2017 Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
@@ -13,15 +13,19 @@ DOCUMENTATION = """
         - This is a helper plugin to allow making other connections persistent.
     version_added: "2.3"
 """
-
-import re
 import os
+import sys
 import pty
+import json
 import subprocess
 
-from ansible.module_utils._text import to_bytes, to_text
-from ansible.module_utils.six.moves import cPickle
+from ansible import constants as C
+from ansible.plugins.loader import connection_loader
 from ansible.plugins.connection import ConnectionBase
+from ansible.module_utils._text import to_text
+from ansible.module_utils.six.moves import cPickle
+from ansible.module_utils.connection import Connection as SocketConnection
+from ansible.errors import AnsibleError
 
 try:
     from __main__ import display
@@ -40,8 +44,38 @@ class Connection(ConnectionBase):
         self._connected = True
         return self
 
-    def _do_it(self, action):
+    def exec_command(self, cmd, in_data=None, sudoable=True):
+        display.vvvv('exec_command(), socket_path=%s' % self.socket_path, host=self._play_context.remote_addr)
+        connection = SocketConnection(self.socket_path)
+        out = connection.exec_command(cmd, in_data=in_data, sudoable=sudoable)
+        return 0, out, ''
 
+    def put_file(self, in_path, out_path):
+        pass
+
+    def fetch_file(self, in_path, out_path):
+        pass
+
+    def close(self):
+        self._connected = False
+
+    def run(self):
+        """Returns the path of the persistent connection socket.
+
+        Attempts to ensure (within playcontext.timeout seconds) that the
+        socket path exists. If the path exists (or the timeout has expired),
+        returns the socket path.
+        """
+        display.vvvv('starting connection from persistent connection plugin', host=self._play_context.remote_addr)
+        socket_path = self._start_connection()
+        display.vvvv('local domain socket path is %s' % socket_path, host=self._play_context.remote_addr)
+        setattr(self, '_socket_path', socket_path)
+        return socket_path
+
+    def _start_connection(self):
+        '''
+        Starts the persistent connection
+        '''
         master, slave = pty.openpty()
         p = subprocess.Popen(["ansible-connection"], stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         stdin = os.fdopen(master, 'wb', 0)
@@ -56,40 +90,23 @@ class Connection(ConnectionBase):
         stdin.write(src)
 
         stdin.write(b'\n#END_INIT#\n')
-        stdin.write(to_bytes(action))
-        stdin.write(b'\n\n')
 
         (stdout, stderr) = p.communicate()
         stdin.close()
 
-        return (p.returncode, stdout, stderr)
+        if p.returncode == 0:
+            result = json.loads(to_text(stdout, errors='surrogate_then_replace'))
+        else:
+            result = json.loads(to_text(stderr, errors='surrogate_then_replace'))
 
-    def exec_command(self, cmd, in_data=None, sudoable=True):
-        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
-        return self._do_it('EXEC: ' + cmd)
+        if 'messages' in result:
+            for msg in result.get('messages'):
+                display.vvvv('%s' % msg, host=self._play_context.remote_addr)
 
-    def put_file(self, in_path, out_path):
-        super(Connection, self).put_file(in_path, out_path)
-        self._do_it('PUT: %s %s' % (in_path, out_path))
+        if 'error' in result:
+            if self._play_context.verbosity > 2:
+                msg = "The full traceback is:\n" + result['exception']
+                display.display(result['exception'], color=C.COLOR_ERROR)
+            raise AnsibleError(result['error'])
 
-    def fetch_file(self, in_path, out_path):
-        super(Connection, self).fetch_file(in_path, out_path)
-        self._do_it('FETCH: %s %s' % (in_path, out_path))
-
-    def close(self):
-        self._connected = False
-
-    def run(self):
-        """Returns the path of the persistent connection socket.
-
-        Attempts to ensure (within playcontext.timeout seconds) that the
-        socket path exists. If the path exists (or the timeout has expired),
-        returns the socket path.
-        """
-        socket_path = None
-        rc, out, err = self._do_it('RUN:')
-        match = re.search(br"#SOCKET_PATH#: (\S+)", out)
-        if match:
-            socket_path = to_text(match.group(1).strip(), errors='surrogate_or_strict')
-
-        return socket_path
+        return result['socket_path']