Add support for Windows hosts in the SSH connection plugin (#47732)

* Add support for Windows hosts in the SSH connection plugin

* fix Python 2.6 unit test and sanity issues

* fix up connection tests in CI, disable SCP for now

* ensure we don't pollute the existing environment during the test

* Add connection_windows_ssh to classifier

* use test dir for inventory file

* Required powershell as default shell and fix tests

* Remove exlicit become_methods on connection

* clarify console encoding comment

* ignore recent SCP errors in integration tests

* Add cmd shell type and added more tests

* Fix some doc issues

* revises windows faq

* add anchors for windows links

* revises windows setup page

* Update changelogs/fragments/windows-ssh.yaml

Co-Authored-By: jborean93 <jborean93@gmail.com>

lib/ansible/plugins/action/__init__.py

@@ -461,7 +461,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
         if remote_user is None:
             remote_user = self._get_remote_user()
 
-        if self._connection._shell.SHELL_FAMILY == 'powershell':
+        if getattr(self._connection._shell, "_IS_WINDOWS", False):
             # This won't work on Powershell as-is, so we'll just completely skip until
             # we have a need for it, at which point we'll have to do something different.
             return remote_paths

lib/ansible/plugins/action/script.py

@@ -65,11 +65,11 @@ class ActionModule(ActionBase):
             chdir = self._task.args.get('chdir')
             if chdir:
                 # Powershell is the only Windows-path aware shell
-                if self._connection._shell.SHELL_FAMILY == 'powershell' and \
+                if getattr(self._connection._shell, "_IS_WINDOWS", False) and \
                         not self.windows_absolute_path_detection.match(chdir):
                     raise AnsibleActionFail('chdir %s must be an absolute path for a Windows remote node' % chdir)
                 # Every other shell is unix-path-aware.
-                if self._connection._shell.SHELL_FAMILY != 'powershell' and not chdir.startswith('/'):
+                if not getattr(self._connection._shell, "_IS_WINDOWS", False) and not chdir.startswith('/'):
                     raise AnsibleActionFail('chdir %s must be an absolute path for a Unix-aware remote node' % chdir)
 
             # Split out the script as the first item in raw_params using
@@ -126,7 +126,7 @@ class ActionModule(ActionBase):
             exec_data = None
             # PowerShell runs the script in a special wrapper to enable things
             # like become and environment args
-            if self._connection._shell.SHELL_FAMILY == "powershell":
+            if getattr(self._connection._shell, "_IS_WINDOWS", False):
                 # FUTURE: use a more public method to get the exec payload
                 pc = self._play_context
                 exec_data = ps_manifest._create_powershell_wrapper(

lib/ansible/plugins/action/wait_for_connection.py

@@ -86,7 +86,7 @@ class ActionModule(ActionBase):
                 pass
 
             # Use win_ping on winrm/powershell, else use ping
-            if hasattr(self._connection, '_shell_type') and self._connection._shell_type == 'powershell':
+            if getattr(self._connection._shell, "_IS_WINDOWS", False):
                 ping_result = self._execute_module(module_name='win_ping', module_args=dict(), task_vars=task_vars)
             else:
                 ping_result = self._execute_module(module_name='ping', module_args=dict(), task_vars=task_vars)

lib/ansible/plugins/connection/__init__.py

@@ -8,6 +8,7 @@ __metaclass__ = type
 import fcntl
 import os
 import shlex
+
 from abc import abstractmethod, abstractproperty
 from functools import wraps
 

lib/ansible/plugins/connection/ssh.py

@@ -276,6 +276,7 @@ import fcntl
 import hashlib
 import os
 import pty
+import re
 import subprocess
 import time
 
@@ -294,6 +295,7 @@ from ansible.module_utils.six.moves import shlex_quote
 from ansible.module_utils._text import to_bytes, to_native, to_text
 from ansible.module_utils.parsing.convert_bool import BOOLEANS, boolean
 from ansible.plugins.connection import ConnectionBase, BUFSIZE
+from ansible.plugins.shell.powershell import _parse_clixml
 from ansible.utils.display import Display
 from ansible.utils.path import unfrackpath, makedirs_safe
 
@@ -453,6 +455,15 @@ class Connection(ConnectionBase):
         self.control_path = C.ANSIBLE_SSH_CONTROL_PATH
         self.control_path_dir = C.ANSIBLE_SSH_CONTROL_PATH_DIR
 
+        # Windows operates differently from a POSIX connection/shell plugin,
+        # we need to set various properties to ensure SSH on Windows continues
+        # to work
+        if getattr(self._shell, "_IS_WINDOWS", False):
+            self.has_native_async = True
+            self.always_pipeline_modules = True
+            self.module_implementation_preferences = ('.ps1', '.exe', '')
+            self.allow_executable = False
+
     # The connection is created by running ssh/scp/sftp from the exec_command,
     # put_file, and fetch_file methods, so we don't need to do any connection
     # management here.
@@ -742,6 +753,7 @@ class Connection(ConnectionBase):
         Starts the command and communicates with it until it ends.
         '''
 
+        # We don't use _shell.quote as this is run on the controller and independent from the shell plugin chosen
         display_cmd = list(map(shlex_quote, map(to_text, cmd)))
         display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host)
 
@@ -1030,6 +1042,12 @@ class Connection(ConnectionBase):
         # accept them for hostnames and IPv4 addresses too.
         host = '[%s]' % self.host
 
+        smart_methods = ['sftp', 'scp', 'piped']
+
+        # Windows does not support dd so we cannot use the piped method
+        if getattr(self._shell, "_IS_WINDOWS", False):
+            smart_methods.remove('piped')
+
         # Transfer methods to try
         methods = []
 
@@ -1039,7 +1057,7 @@ class Connection(ConnectionBase):
             if not (ssh_transfer_method in ('smart', 'sftp', 'scp', 'piped')):
                 raise AnsibleOptionsError('transfer_method needs to be one of [smart|sftp|scp|piped]')
             if ssh_transfer_method == 'smart':
-                methods = ['sftp', 'scp', 'piped']
+                methods = smart_methods
             else:
                 methods = [ssh_transfer_method]
         else:
@@ -1052,7 +1070,7 @@ class Connection(ConnectionBase):
                 elif scp_if_ssh != 'smart':
                     raise AnsibleOptionsError('scp_if_ssh needs to be one of [smart|True|False]')
             if scp_if_ssh == 'smart':
-                methods = ['sftp', 'scp', 'piped']
+                methods = smart_methods
             elif scp_if_ssh is True:
                 methods = ['scp']
             else:
@@ -1067,10 +1085,11 @@ class Connection(ConnectionBase):
                 (returncode, stdout, stderr) = self._bare_run(cmd, in_data, checkrc=False)
             elif method == 'scp':
                 scp = self.get_option('scp_executable')
+
                 if sftp_action == 'get':
-                    cmd = self._build_command(scp, u'{0}:{1}'.format(host, shlex_quote(in_path)), out_path)
+                    cmd = self._build_command(scp, u'{0}:{1}'.format(host, self._shell.quote(in_path)), out_path)
                 else:
-                    cmd = self._build_command(scp, in_path, u'{0}:{1}'.format(host, shlex_quote(out_path)))
+                    cmd = self._build_command(scp, in_path, u'{0}:{1}'.format(host, self._shell.quote(out_path)))
                 in_data = None
                 (returncode, stdout, stderr) = self._bare_run(cmd, in_data, checkrc=False)
             elif method == 'piped':
@@ -1105,6 +1124,16 @@ class Connection(ConnectionBase):
             raise AnsibleError("failed to transfer file to %s %s:\n%s\n%s" %
                                (to_native(in_path), to_native(out_path), to_native(stdout), to_native(stderr)))
 
+    def _escape_win_path(self, path):
+        """ converts a Windows path to one that's supported by SFTP and SCP """
+        # If using a root path then we need to start with /
+        prefix = ""
+        if re.match(r'^\w{1}:', path):
+            prefix = "/"
+
+        # Convert all '\' to '/'
+        return "%s%s" % (prefix, path.replace("\\", "/"))
+
     #
     # Main public methods
     #
@@ -1115,6 +1144,18 @@ class Connection(ConnectionBase):
 
         display.vvv(u"ESTABLISH SSH CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self._play_context.remote_addr)
 
+        if getattr(self._shell, "_IS_WINDOWS", False):
+            # Become method 'runas' is done in the wrapper that is executed,
+            # need to disable sudoable so the bare_run is not waiting for a
+            # prompt that will not occur
+            sudoable = False
+
+            # Make sure our first command is to set the console encoding to
+            # utf-8, this must be done via chcp to get utf-8 (65001)
+            cmd_parts = ["chcp.com", "65001", self._shell._SHELL_REDIRECT_ALLNULL, self._shell._SHELL_AND]
+            cmd_parts.extend(self._shell._encode_script(cmd, as_list=True, strict_mode=False, preserve_rc=False))
+            cmd = ' '.join(cmd_parts)
+
         # we can only use tty when we are not pipelining the modules. piping
         # data into /usr/bin/python inside a tty automatically invokes the
         # python interactive-mode but the modules are not compatible with the
@@ -1134,6 +1175,10 @@ class Connection(ConnectionBase):
         cmd = self._build_command(*args)
         (returncode, stdout, stderr) = self._run(cmd, in_data, sudoable=sudoable)
 
+        # When running on Windows, stderr may contain CLIXML encoded output
+        if getattr(self._shell, "_IS_WINDOWS", False) and stderr.startswith(b"#< CLIXML"):
+            stderr = _parse_clixml(stderr)
+
         return (returncode, stdout, stderr)
 
     def put_file(self, in_path, out_path):
@@ -1145,6 +1190,9 @@ class Connection(ConnectionBase):
         if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
             raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_native(in_path)))
 
+        if getattr(self._shell, "_IS_WINDOWS", False):
+            out_path = self._escape_win_path(out_path)
+
         return self._file_transport_command(in_path, out_path, 'put')
 
     def fetch_file(self, in_path, out_path):
@@ -1153,6 +1201,11 @@ class Connection(ConnectionBase):
         super(Connection, self).fetch_file(in_path, out_path)
 
         display.vvv(u"FETCH {0} TO {1}".format(in_path, out_path), host=self.host)
+
+        # need to add / if path is rooted
+        if getattr(self._shell, "_IS_WINDOWS", False):
+            in_path = self._escape_win_path(in_path)
+
         return self._file_transport_command(in_path, out_path, 'get')
 
     def reset(self):

lib/ansible/plugins/connection/winrm.py

@@ -104,7 +104,6 @@ import traceback
 import json
 import tempfile
 import subprocess
-import xml.etree.ElementTree as ET
 
 HAVE_KERBEROS = False
 try:
@@ -122,6 +121,7 @@ from ansible.module_utils.six.moves.urllib.parse import urlunsplit
 from ansible.module_utils._text import to_bytes, to_native, to_text
 from ansible.module_utils.six import binary_type, PY3
 from ansible.plugins.connection import ConnectionBase
+from ansible.plugins.shell.powershell import _parse_clixml
 from ansible.utils.hashing import secure_hash
 from ansible.utils.path import makedirs_safe
 from ansible.utils.display import Display
@@ -538,28 +538,15 @@ class Connection(ConnectionBase):
         result.std_err = to_bytes(result.std_err)
 
         # parse just stderr from CLIXML output
-        if self.is_clixml(result.std_err):
+        if result.std_err.startswith(b"#< CLIXML"):
             try:
-                result.std_err = self.parse_clixml_stream(result.std_err)
+                result.std_err = _parse_clixml(result.std_err)
             except Exception:
                 # unsure if we're guaranteed a valid xml doc- use raw output in case of error
                 pass
 
         return (result.status_code, result.std_out, result.std_err)
 
-    def is_clixml(self, value):
-        return value.startswith(b"#< CLIXML\r\n")
-
-    # hacky way to get just stdout- not always sure of doc framing here, so use with care
-    def parse_clixml_stream(self, clixml_doc, stream_name='Error'):
-        clixml = ET.fromstring(clixml_doc.split(b"\r\n", 1)[-1])
-        namespace_match = re.match(r'{(.*)}', clixml.tag)
-        namespace = "{%s}" % namespace_match.group(1) if namespace_match else ""
-
-        strings = clixml.findall("./%sS" % namespace)
-        lines = [e.text.replace('_x000D__x000A_', '') for e in strings if e.attrib.get('S') == stream_name]
-        return to_bytes('\r\n'.join(lines))
-
     # FUTURE: determine buffer size at runtime via remote winrm config?
     def _put_file_stdin_iterator(self, in_path, out_path, buffer_size=250000):
         in_size = os.path.getsize(to_bytes(in_path, errors='surrogate_or_strict'))

lib/ansible/plugins/doc_fragments/shell_windows.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2019 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+
+class ModuleDocFragment(object):
+
+    # Windows shell documentation fragment
+    # FIXME: set_module_language don't belong here but must be set so they don't fail when someone
+    #  get_option('set_module_language') on this plugin
+    DOCUMENTATION = """
+options:
+  async_dir:
+    description:
+    - Directory in which ansible will keep async job information.
+    - Before Ansible 2.8, this was set to C(remote_tmp + "\\.ansible_async").
+    default: '%USERPROFILE%\\.ansible_async'
+    ini:
+    - section: powershell
+      key: async_dir
+    vars:
+    - name: ansible_async_dir
+    version_added: '2.8'
+  remote_tmp:
+    description:
+    - Temporary directory to use on targets when copying files to the host.
+    default: '%TEMP%'
+    ini:
+    - section: powershell
+      key: remote_tmp
+    vars:
+    - name: ansible_remote_tmp
+  set_module_language:
+    description:
+    - Controls if we set the locale for moduels when executing on the
+      target.
+    - Windows only supports C(no) as an option.
+    type: bool
+    default: 'no'
+    choices:
+    - 'no'
+  environment:
+    description:
+    - Dictionary of environment variables and their values to use when
+      executing commands.
+    type: dict
+    default: {}
+"""

lib/ansible/plugins/shell/__init__.py

@@ -221,3 +221,7 @@ class ShellBase(AnsiblePlugin):
     def wrap_for_exec(self, cmd):
         """wrap script execution with any necessary decoration (eg '&' for quoted powershell script paths)"""
         return cmd
+
+    def quote(self, cmd):
+        """Returns a shell-escaped string that can be safely used as one token in a shell command line"""
+        return shlex_quote(cmd)

lib/ansible/plugins/shell/cmd.py

@@ -0,0 +1,57 @@
+# Copyright (c) 2019 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+name: cmd
+plugin_type: shell
+version_added: '2.8'
+short_description: Windows Command Prompt
+description:
+- Used with the 'ssh' connection plugin and no C(DefaultShell) has been set on the Windows host.
+extends_documentation_fragment:
+- shell_windows
+'''
+
+import re
+
+from ansible.plugins.shell.powershell import ShellModule as PSShellModule
+
+# these are the metachars that have a special meaning in cmd that we want to escape when quoting
+_find_unsafe = re.compile(r'[\s\(\)\%\!^\"\<\>\&\|]').search
+
+
+class ShellModule(PSShellModule):
+
+    # Common shell filenames that this plugin handles
+    COMPATIBLE_SHELLS = frozenset()
+    # Family of shells this has.  Must match the filename without extension
+    SHELL_FAMILY = 'cmd'
+
+    _SHELL_REDIRECT_ALLNULL = '>nul 2>&1'
+    _SHELL_AND = '&&'
+
+    # Used by various parts of Ansible to do Windows specific changes
+    _IS_WINDOWS = True
+
+    def quote(self, s):
+        # cmd does not support single quotes that the shlex_quote uses. We need to override the quoting behaviour to
+        # better match cmd.exe.
+        # https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/
+
+        # Return an empty argument
+        if not s:
+            return '""'
+
+        if _find_unsafe(s) is None:
+            return s
+
+        # Escape the metachars as we are quoting the string to stop cmd from interpreting that metachar. For example
+        # 'file &whoami.exe' would result in 'file $(whoami.exe)' instead of the literal string
+        # https://stackoverflow.com/questions/3411771/multiple-character-replace-with-python
+        for c in '^()%!"<>&|':  # '^' must be the first char that we scan and replace
+            if c in s:
+                s = s.replace(c, "^" + c)
+
+        return '^"' + s + '^"'

lib/ansible/plugins/shell/powershell.py

@@ -5,60 +5,26 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 DOCUMENTATION = '''
-    name: powershell
-    plugin_type: shell
-    version_added: ""
-    short_description: Windows Powershell
-    description:
-      - The only option when using 'winrm' as a connection plugin
-    options:
-      async_dir:
-        description:
-        - Directory in which ansible will keep async job information.
-        - Before Ansible 2.8, this was set to C(remote_tmp + "\\.ansible_async").
-        default: '%USERPROFILE%\\.ansible_async'
-        ini:
-        - section: powershell
-          key: async_dir
-        vars:
-        - name: ansible_async_dir
-        version_added: '2.8'
-      remote_tmp:
-        description:
-        - Temporary directory to use on targets when copying files to the host.
-        default: '%TEMP%'
-        ini:
-        - section: powershell
-          key: remote_tmp
-        vars:
-        - name: ansible_remote_tmp
-      set_module_language:
-        description:
-        - Controls if we set the locale for moduels when executing on the
-          target.
-        - Windows only supports C(no) as an option.
-        type: bool
-        default: 'no'
-        choices:
-        - 'no'
-      environment:
-        description:
-        - Dictionary of environment variables and their values to use when
-          executing commands.
-        type: dict
-        default: {}
+name: powershell
+plugin_type: shell
+version_added: historical
+short_description: Windows PowerShell
+description:
+- The only option when using 'winrm' or 'psrp' as a connection plugin.
+- Can also be used when using 'ssh' as a connection plugin and the C(DefaultShell) has been configured to PowerShell.
+extends_documentation_fragment:
+- shell_windows
 '''
-# FIXME: admin_users and set_module_language don't belong here but must be set
-# so they don't failk when someone get_option('admin_users') on this plugin
 
 import base64
 import os
 import re
 import shlex
 import pkgutil
+import xml.etree.ElementTree as ET
 
 from ansible.errors import AnsibleError
-from ansible.module_utils._text import to_text
+from ansible.module_utils._text import to_bytes, to_text
 from ansible.plugins.shell import ShellBase
 
 
@@ -71,6 +37,21 @@ if _powershell_version:
     _common_args = ['PowerShell', '-Version', _powershell_version] + _common_args[1:]
 
 
+def _parse_clixml(data, stream="Error"):
+    """
+    Takes a byte string like '#< CLIXML\r\n<Objs...' and extracts the stream
+    message encoded in the XML data. CLIXML is used by PowerShell to encode
+    multiple objects in stderr.
+    """
+    clixml = ET.fromstring(data.split(b"\r\n", 1)[-1])
+    namespace_match = re.match(r'{(.*)}', clixml.tag)
+    namespace = "{%s}" % namespace_match.group(1) if namespace_match else ""
+
+    strings = clixml.findall("./%sS" % namespace)
+    lines = [e.text.replace('_x000D__x000A_', '') for e in strings if e.attrib.get('S') == stream]
+    return to_bytes('\r\n'.join(lines))
+
+
 class ShellModule(ShellBase):
 
     # Common shell filenames that this plugin handles
@@ -80,6 +61,12 @@ class ShellModule(ShellBase):
     # Family of shells this has.  Must match the filename without extension
     SHELL_FAMILY = 'powershell'
 
+    _SHELL_REDIRECT_ALLNULL = '> $null'
+    _SHELL_AND = ';'
+
+    # Used by various parts of Ansible to do Windows specific changes
+    _IS_WINDOWS = True
+
     env = dict()
 
     # We're being overly cautious about which keys to accept (more so than