Creating playbook executor and dependent classes

v2/ansible/plugins/__init__.py

@@ -240,7 +240,7 @@ callback_loader = PluginLoader(
 
 connection_loader = PluginLoader(
     'Connection',
-    'ansible.plugins.connection',
+    'ansible.plugins.connections',
     C.DEFAULT_CONNECTION_PLUGIN_PATH,
     'connection_plugins',
     aliases={'paramiko': 'paramiko_ssh'}
@@ -253,37 +253,44 @@ shell_loader = PluginLoader(
     'shell_plugins',
 )
 
-module_finder = PluginLoader(
+module_loader = PluginLoader(
     '',
     'ansible.modules',
     C.DEFAULT_MODULE_PATH,
     'library'
 )
 
-lookup_finder = PluginLoader(
+lookup_loader = PluginLoader(
     'LookupModule',
     'ansible.plugins.lookup',
     C.DEFAULT_LOOKUP_PLUGIN_PATH,
     'lookup_plugins'
 )
 
-vars_finder = PluginLoader(
+vars_loader = PluginLoader(
     'VarsModule',
     'ansible.plugins.vars',
     C.DEFAULT_VARS_PLUGIN_PATH,
     'vars_plugins'
 )
 
-filter_finder = PluginLoader(
+filter_loader = PluginLoader(
     'FilterModule',
     'ansible.plugins.filter',
     C.DEFAULT_FILTER_PLUGIN_PATH,
     'filter_plugins'
 )
 
-fragment_finder = PluginLoader(
+fragment_loader = PluginLoader(
     'ModuleDocFragment',
     'ansible.utils.module_docs_fragments',
     os.path.join(os.path.dirname(__file__), 'module_docs_fragments'),
     '',
 )
+
+strategy_loader = PluginLoader(
+    'StrategyModule',
+    'ansible.plugins.strategies',
+    None,
+    'strategy_plugins',
+)

v2/ansible/plugins/action/__init__.py

@@ -19,3 +19,410 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import StringIO
+import json
+import os
+import random
+import tempfile
+import time
+
+from ansible import constants as C
+from ansible.errors import AnsibleError
+from ansible.executor.module_common import ModuleReplacer
+from ansible.parsing.utils.jsonify import jsonify
+from ansible.plugins import module_loader, shell_loader
+
+from ansible.utils.debug import debug
+
+class ActionBase:
+
+    '''
+    This class is the base class for all action plugins, and defines
+    code common to all actions. The base class handles the connection
+    by putting/getting files and executing commands based on the current
+    action in use.
+    '''
+
+    def __init__(self, task, connection, connection_info, loader):
+        self._task            = task
+        self._connection      = connection
+        self._connection_info = connection_info
+        self._loader          = loader
+        self._shell           = self.get_shell()
+
+    def get_shell(self):
+
+        # FIXME: no more inject, get this from the host variables?
+        #default_shell = getattr(self._connection, 'default_shell', '')
+        #shell_type = inject.get('ansible_shell_type')
+        #if not shell_type:
+        #    if default_shell:
+        #        shell_type = default_shell
+        #    else:
+        #        shell_type = os.path.basename(C.DEFAULT_EXECUTABLE)
+
+        shell_type = getattr(self._connection, 'default_shell', '')
+        if not shell_type:
+            shell_type = os.path.basename(C.DEFAULT_EXECUTABLE)
+
+        shell_plugin = shell_loader.get(shell_type)
+        if shell_plugin is None:
+            shell_plugin = shell_loader.get('sh')
+
+        return shell_plugin
+
+    def _configure_module(self, module_name, module_args):
+        '''
+        Handles the loading and templating of the module code through the
+        ModuleReplacer class.
+        '''
+
+        # Search module path(s) for named module.
+        module_suffixes = getattr(self._connection, 'default_suffixes', None)
+        module_path = module_loader.find_plugin(module_name, module_suffixes, transport=self._connection.get_transport())
+        if module_path is None:
+            module_path2 = module_loader.find_plugin('ping', module_suffixes)
+            if module_path2 is not None:
+                raise AnsibleError("The module %s was not found in configured module paths" % (module_name))
+            else:
+                raise AnsibleError("The module %s was not found in configured module paths. " \
+                                   "Additionally, core modules are missing. If this is a checkout, " \
+                                   "run 'git submodule update --init --recursive' to correct this problem." % (module_name))
+
+        # insert shared code and arguments into the module
+        (module_data, module_style, module_shebang) = ModuleReplacer().modify_module(module_path, module_args)
+
+        return (module_style, module_shebang, module_data)
+
+    def _compute_environment_string(self):
+        '''
+        Builds the environment string to be used when executing the remote task.
+        '''
+
+        enviro = {}
+
+        # FIXME: not sure where this comes from, probably task but maybe also the play?
+        #if self.environment:
+        #    enviro = template.template(self.basedir, self.environment, inject, convert_bare=True)
+        #    enviro = utils.safe_eval(enviro)
+        #    if type(enviro) != dict:
+        #        raise errors.AnsibleError("environment must be a dictionary, received %s" % enviro)
+
+        return self._shell.env_prefix(**enviro)
+
+    def _early_needs_tmp_path(self):
+        '''
+        Determines if a temp path should be created before the action is executed.
+        '''
+
+        # FIXME: modified from original, needs testing? Since this is now inside
+        #        the action plugin, it should make it just this simple
+        return getattr(self, 'TRANSFERS_FILES', False)
+        
+    def _late_needs_tmp_path(self, tmp, module_style):
+        '''
+        Determines if a temp path is required after some early actions have already taken place.
+        '''
+        if tmp and "tmp" in tmp:
+            # tmp has already been created
+            return False
+        if not self._connection._has_pipelining or not C.ANSIBLE_SSH_PIPELINING or C.DEFAULT_KEEP_REMOTE_FILES or self._connection_info.su:
+            # tmp is necessary to store module source code
+            return True
+        if not self._connection._has_pipelining:
+            # tmp is necessary to store the module source code
+            # or we want to keep the files on the target system
+            return True
+        if module_style != "new":
+            # even when conn has pipelining, old style modules need tmp to store arguments
+            return True
+        return False
+
+    # FIXME: return a datastructure in this function instead of raising errors -
+    #        the new executor pipeline handles it much better that way
+    def _make_tmp_path(self):
+        '''
+        Create and return a temporary path on a remote box.
+        '''
+
+        basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(0, 2**48))
+        use_system_tmp = False
+
+        if (self._connection_info.sudo and self._connection_info.sudo_user != 'root') or (self._connection_info.su and self._connection_info.su_user != 'root'):
+            use_system_tmp = True
+
+        tmp_mode = None
+        if self._connection_info.remote_user != 'root' or \
+           ((self._connection_info.sudo and self._connection_info.sudo_user != 'root') or (self._connection_info.su and self._connection_info.su_user != 'root')):
+            tmp_mode = 'a+rx'
+
+        cmd = self._shell.mkdtemp(basefile, use_system_tmp, tmp_mode)
+        result = self._low_level_execute_command(cmd, None, sudoable=False)
+
+        # error handling on this seems a little aggressive?
+        if result['rc'] != 0:
+            if result['rc'] == 5:
+                output = 'Authentication failure.'
+            elif result['rc'] == 255 and self._connection.get_transport() in ['ssh']:
+                # FIXME: more utils.VERBOSITY
+                #if utils.VERBOSITY > 3:
+                #    output = 'SSH encountered an unknown error. The output was:\n%s' % (result['stdout']+result['stderr'])
+                #else:
+                #    output = 'SSH encountered an unknown error during the connection. We recommend you re-run the command using -vvvv, which will enable SSH debugging output to help diagnose the issue'
+                output = 'SSH encountered an unknown error. The output was:\n%s' % (result['stdout']+result['stderr'])
+            elif 'No space left on device' in result['stderr']:
+                output = result['stderr']
+            else:
+                output = 'Authentication or permission failure.  In some cases, you may have been able to authenticate and did not have permissions on the remote directory. Consider changing the remote temp path in ansible.cfg to a path rooted in "/tmp". Failed command was: %s, exited with result %d' % (cmd, result['rc'])
+            if 'stdout' in result and result['stdout'] != '':
+                output = output + ": %s" % result['stdout']
+            raise AnsibleError(output)
+
+        # FIXME: do we still need to do this?
+        #rc = self._shell.join_path(utils.last_non_blank_line(result['stdout']).strip(), '')
+        rc = self._shell.join_path(result['stdout'].strip(), '').splitlines()[-1]
+
+        # Catch failure conditions, files should never be
+        # written to locations in /.
+        if rc == '/':
+            raise AnsibleError('failed to resolve remote temporary directory from %s: `%s` returned empty string' % (basetmp, cmd))
+
+        return rc
+
+    def _remove_tmp_path(self, tmp_path):
+        '''Remove a temporary path we created. '''
+
+        if "-tmp-" in tmp_path:
+            cmd = self._shell.remove(tmp_path, recurse=True)
+            # If we have gotten here we have a working ssh configuration.
+            # If ssh breaks we could leave tmp directories out on the remote system.
+            self._low_level_execute_command(cmd, None, sudoable=False)
+
+    def _transfer_data(self, remote_path, data):
+        '''
+        Copies the module data out to the temporary module path.
+        '''
+
+        if type(data) == dict:
+            data = jsonify(data)
+
+        afd, afile = tempfile.mkstemp()
+        afo = os.fdopen(afd, 'w')
+        try:
+            if not isinstance(data, unicode):
+                #ensure the data is valid UTF-8
+                data = data.decode('utf-8')
+            else:
+                data = data.encode('utf-8')
+            afo.write(data)
+        except Exception, e:
+            raise AnsibleError("failure encoding into utf-8: %s" % str(e))
+
+        afo.flush()
+        afo.close()
+
+        try:
+            self._connection.put_file(afile, remote_path)
+        finally:
+            os.unlink(afile)
+
+        return remote_path
+
+    def _remote_chmod(self, tmp, mode, path, sudoable=False):
+        '''
+        Issue a remote chmod command
+        '''
+
+        cmd = self._shell.chmod(mode, path)
+        return self._low_level_execute_command(cmd, tmp, sudoable=sudoable)
+
+    def _remote_checksum(self, tmp, path):
+        '''
+        Takes a remote checksum and returns 1 if no file
+        '''
+
+        # FIXME: figure out how this will work, probably pulled from the
+        #        variable manager data
+        #python_interp = inject['hostvars'][inject['inventory_hostname']].get('ansible_python_interpreter', 'python')
+        python_interp = 'python'
+        cmd = self._shell.checksum(path, python_interp)
+        data = self._low_level_execute_command(cmd, tmp, sudoable=True)
+        # FIXME: implement this function?
+        #data2 = utils.last_non_blank_line(data['stdout'])
+        data2 = data['stdout'].strip().splitlines()[-1]
+        try:
+            if data2 == '':
+                # this may happen if the connection to the remote server
+                # failed, so just return "INVALIDCHECKSUM" to avoid errors
+                return "INVALIDCHECKSUM"
+            else:
+                return data2.split()[0]
+        except IndexError:
+            sys.stderr.write("warning: Calculating checksum failed unusually, please report this to the list so it can be fixed\n")
+            sys.stderr.write("command: %s\n" % cmd)
+            sys.stderr.write("----\n")
+            sys.stderr.write("output: %s\n" % data)
+            sys.stderr.write("----\n")
+            # this will signal that it changed and allow things to keep going
+            return "INVALIDCHECKSUM"
+
+    def _remote_expand_user(self, path, tmp):
+        ''' takes a remote path and performs tilde expansion on the remote host '''
+        if not path.startswith('~'):
+            return path
+
+        split_path = path.split(os.path.sep, 1)
+        expand_path = split_path[0]
+        if expand_path == '~':
+            if self._connection_info.sudo and self._connection_info.sudo_user:
+                expand_path = '~%s' % self._connection_info.sudo_user
+            elif self._connection_info.su and self._connection_info.su_user:
+                expand_path = '~%s' % self._connection_info.su_user
+
+        cmd = self._shell.expand_user(expand_path)
+        data = self._low_level_execute_command(cmd, tmp, sudoable=False)
+        #initial_fragment = utils.last_non_blank_line(data['stdout'])
+        initial_fragment = data['stdout'].strip().splitlines()[-1]
+
+        if not initial_fragment:
+            # Something went wrong trying to expand the path remotely.  Return
+            # the original string
+            return path
+
+        if len(split_path) > 1:
+            return self._shell.join_path(initial_fragment, *split_path[1:])
+        else:
+            return initial_fragment
+
+    def _filter_leading_non_json_lines(self, data):
+        '''
+        Used to avoid random output from SSH at the top of JSON output, like messages from
+        tcagetattr, or where dropbear spews MOTD on every single command (which is nuts).
+
+        need to filter anything which starts not with '{', '[', ', '=' or is an empty line.
+        filter only leading lines since multiline JSON is valid.
+        '''
+
+        filtered_lines = StringIO.StringIO()
+        stop_filtering = False
+        for line in data.splitlines():
+            if stop_filtering or line.startswith('{') or line.startswith('['):
+                stop_filtering = True
+                filtered_lines.write(line + '\n')
+        return filtered_lines.getvalue()
+
+    def _execute_module(self, module_name=None, module_args=None, tmp=None, persist_files=False, delete_remote_tmp=True):
+        '''
+        Transfer and run a module along with its arguments.
+        '''
+
+        # if a module name was not specified for this execution, use
+        # the action from the task
+        if module_name is None:
+            module_name = self._task.action
+        if module_args is None:
+            module_args = self._task.args
+
+        debug("in _execute_module (%s, %s)" % (module_name, module_args))
+
+        (module_style, shebang, module_data) = self._configure_module(module_name=module_name, module_args=module_args)
+        if not shebang:
+            raise AnsibleError("module is missing interpreter line")
+
+        # a remote tmp path may be necessary and not already created
+        remote_module_path = None
+        if self._late_needs_tmp_path(tmp, module_style):
+            tmp = self._make_tmp_path()
+            remote_module_path = self._shell.join_path(tmp, module_name)
+
+        # FIXME: async stuff here
+        #if (module_style != 'new' or async_jid is not None or not self._connection._has_pipelining or not C.ANSIBLE_SSH_PIPELINING or C.DEFAULT_KEEP_REMOTE_FILES):
+        if remote_module_path:
+            self._transfer_data(remote_module_path, module_data)
+
+        environment_string = self._compute_environment_string()
+
+        if tmp and "tmp" in tmp and ((self._connection_info.sudo and self._connection_info.sudo_user != 'root') or (self._connection_info.su and self._connection_info.su_user != 'root')):
+            # deal with possible umask issues once sudo'ed to other user
+            self._remote_chmod(tmp, 'a+r', remote_module_path)
+
+        cmd = ""
+        in_data = None
+
+        # FIXME: all of the old-module style and async stuff has been removed from here, and
+        #        might need to be re-added (unless we decide to drop support for old-style modules
+        #        at this point and rework things to support non-python modules specifically)
+        if self._connection._has_pipelining and C.ANSIBLE_SSH_PIPELINING and not C.DEFAULT_KEEP_REMOTE_FILES:
+            in_data = module_data
+        else:
+            if remote_module_path:
+                cmd = remote_module_path
+
+        rm_tmp = None
+        if tmp and "tmp" in tmp and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files and delete_remote_tmp:
+            if not self._connection_info.sudo or self._connection_info.su or self._connection_info.sudo_user == 'root' or self._connection_info.su_user == 'root':
+                # not sudoing or sudoing to root, so can cleanup files in the same step
+                rm_tmp = tmp
+
+        cmd = self._shell.build_module_command(environment_string, shebang, cmd, rm_tmp)
+        cmd = cmd.strip()
+
+        sudoable = True
+        if module_name == "accelerate":
+            # always run the accelerate module as the user
+            # specified in the play, not the sudo_user
+            sudoable = False
+
+        res = self._low_level_execute_command(cmd, tmp, sudoable=sudoable, in_data=in_data)
+
+        if tmp and "tmp" in tmp and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files and delete_remote_tmp:
+            if (self._connection_info.sudo and self._connection_info.sudo_user != 'root') or (self._connection_info.su and self._connection_info.su_user != 'root'):
+            # not sudoing to root, so maybe can't delete files as that other user
+            # have to clean up temp files as original user in a second step
+                cmd2 = self._shell.remove(tmp, recurse=True)
+                self._low_level_execute_command(cmd2, tmp, sudoable=False)
+
+        # FIXME: in error situations, the stdout may not contain valid data, so we
+        #        should check for bad rc codes better to catch this here
+        data = json.loads(self._filter_leading_non_json_lines(res['stdout']))
+        if 'parsed' in data and data['parsed'] == False:
+            data['msg'] += res['stderr']
+
+        debug("done with _execute_module (%s, %s)" % (module_name, module_args))
+        return data
+
+    def _low_level_execute_command(self, cmd, tmp, executable=None, sudoable=False, in_data=None):
+        '''
+        This is the function which executes the low level shell command, which
+        may be commands to create/remove directories for temporary files, or to
+        run the module code or python directly when pipelining.
+        '''
+
+        debug("in _low_level_execute_command() (%s)" % (cmd,))
+        if not cmd:
+            # this can happen with powershell modules when there is no analog to a Windows command (like chmod)
+            debug("no command, exiting _low_level_execute_command()")
+            return dict(stdout='', stderr='')
+
+        if executable is None:
+            executable = C.DEFAULT_EXECUTABLE
+
+        debug("executing the command through the connection")
+        rc, stdin, stdout, stderr = self._connection.exec_command(cmd, tmp, executable=executable, in_data=in_data, sudoable=sudoable)
+        debug("command execution done")
+
+        if not isinstance(stdout, basestring):
+            out = ''.join(stdout.readlines())
+        else:
+            out = stdout
+
+        if not isinstance(stderr, basestring):
+            err = ''.join(stderr.readlines())
+        else:
+            err = stderr
+
+        debug("done with _low_level_execute_command() (%s)" % (cmd,))
+        if rc is not None:
+            return dict(rc=rc, stdout=out, stderr=err)
+        else:
+            return dict(stdout=out, stderr=err)

v2/ansible/plugins/action/assemble.py

@@ -0,0 +1,159 @@
+# (c) 2013-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#           Stephen Fromm <sfromm@gmail.com>
+#           Brian Coca  <briancoca+dev@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+
+import os
+import os.path
+import pipes
+import shutil
+import tempfile
+import base64
+import re
+
+
+from ansible.plugins.action import ActionBase
+from ansible.utils.hashing import checksum_s
+
+class ActionModule(ActionBase):
+
+    TRANSFERS_FILES = True
+
+    def _assemble_from_fragments(self, src_path, delimiter=None, compiled_regexp=None):
+        ''' assemble a file from a directory of fragments '''
+
+        tmpfd, temp_path = tempfile.mkstemp()
+        tmp = os.fdopen(tmpfd,'w')
+        delimit_me = False
+        add_newline = False
+
+        for f in sorted(os.listdir(src_path)):
+            if compiled_regexp and not compiled_regexp.search(f):
+                continue
+            fragment = "%s/%s" % (src_path, f)
+            if not os.path.isfile(fragment):
+                continue
+            fragment_content = file(fragment).read()
+
+            # always put a newline between fragments if the previous fragment didn't end with a newline.
+            if add_newline:
+                tmp.write('\n')
+
+            # delimiters should only appear between fragments
+            if delimit_me:
+                if delimiter:
+                    # un-escape anything like newlines
+                    delimiter = delimiter.decode('unicode-escape')
+                    tmp.write(delimiter)
+                    # always make sure there's a newline after the
+                    # delimiter, so lines don't run together
+                    if delimiter[-1] != '\n':
+                        tmp.write('\n')
+
+            tmp.write(fragment_content)
+            delimit_me = True
+            if fragment_content.endswith('\n'):
+                add_newline = False
+            else:
+                add_newline = True
+
+        tmp.close()
+        return temp_path
+
+    def run(self, tmp=None, task_vars=dict()):
+
+        src        = self._task.args.get('src', None)
+        dest       = self._task.args.get('dest', None)
+        delimiter  = self._task.args.get('delimiter', None)
+        # FIXME: boolean needs to be moved out of utils
+        #remote_src = utils.boolean(options.get('remote_src', 'yes'))
+        remote_src = self._task.args.get('remote_src', 'yes')
+        regexp     = self._task.args.get('regexp', None)
+
+        if src is None or dest is None:
+            return dict(failed=True, msg="src and dest are required")
+
+        # FIXME: this should be boolean, hard-coded to yes for testing
+        if remote_src == 'yes':
+            return self._execute_module(tmp=tmp)
+        # FIXME: we don't do inject anymore, so not sure where the original
+        #        file stuff is going to end up at this time
+        #elif '_original_file' in inject:
+        #    src = utils.path_dwim_relative(inject['_original_file'], 'files', src, self.runner.basedir)
+        else:
+            # the source is local, so expand it here
+            src = os.path.expanduser(src)
+
+        _re = None
+        if regexp is not None:
+            _re = re.compile(regexp)
+
+        # Does all work assembling the file
+        path = self._assemble_from_fragments(src, delimiter, _re)
+
+        path_checksum = checksum_s(path)
+        dest = self._remote_expand_user(dest, tmp)
+        remote_checksum = self._remote_checksum(tmp, dest)
+
+        if path_checksum != remote_checksum:
+            resultant = file(path).read()
+            # FIXME: diff needs to be moved somewhere else
+            #if self.runner.diff:
+            #    dest_result = self._execute_module(module_name='slurp', module_args=dict(path=dest), tmp=tmp, persist_files=True)
+            #    if 'content' in dest_result:
+            #        dest_contents = dest_result['content']
+            #        if dest_result['encoding'] == 'base64':
+            #            dest_contents = base64.b64decode(dest_contents)
+            #        else:
+            #            raise Exception("unknown encoding, failed: %s" % dest_result)
+            xfered = self._transfer_data('src', resultant)
+
+            # fix file permissions when the copy is done as a different user
+            if self._connection_info.sudo and self._connection_info.sudo_user != 'root' or self._connection_info.su and self._connection_info.su_user != 'root':
+                self._remote_chmod('a+r', xfered, tmp)
+
+            # run the copy module
+            
+            new_module_args = self._task.args.copy()
+            new_module_args.update(
+                dict(
+                    src=xfered,
+                    dest=dest,
+                    original_basename=os.path.basename(src),
+                )
+            )
+
+            # FIXME: checkmode stuff
+            #if self.runner.noop_on_check(inject):
+            #    return ReturnData(conn=conn, comm_ok=True, result=dict(changed=True), diff=dict(before_header=dest, after_header=src, after=resultant))
+            #else:
+            #    res = self.runner._execute_module(conn, tmp, 'copy', module_args_tmp, inject=inject)
+            #    res.diff = dict(after=resultant)
+            #    return res
+            res = self._execute_module(module_name='copy', module_args=new_module_args, tmp=tmp)
+            #res.diff = dict(after=resultant)
+            return res
+        else:
+            new_module_args = self._task.args.copy()
+            new_module_args.update(
+                dict(
+                    src=xfered,
+                    dest=dest,
+                    original_basename=os.path.basename(src),
+                )
+            )
+
+            return self._execute_module(module_name='file', module_args=new_module_args, tmp=tmp)

v2/ansible/plugins/action/assert.py

@@ -0,0 +1,54 @@
+# Copyright 2012, Dag Wieers <dag@wieers.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from ansible.errors import AnsibleError
+from ansible.plugins.action import ActionBase
+
+class ActionModule(ActionBase):
+    ''' Fail with custom message '''
+
+    TRANSFERS_FILES = False
+
+    def run(self, tmp=None, task_vars=dict()):
+
+        # note: the fail module does not need to pay attention to check mode
+        # it always runs.
+
+        msg = None
+        if 'msg' in self._task.args:
+            msg = self._task.args['msg']
+
+        if not 'that' in self._task.args:
+            raise AnsibleError('conditional required in "that" string')
+
+        for that in self._task.args['that']:
+            self._task.when = [ that ]
+            test_result = self._task.evaluate_conditional(all_vars=task_vars)
+            if not test_result:
+                result = dict(
+                   failed       = True,
+                   evaluated_to = test_result,
+                   assertion    = that,
+                )
+
+                if msg:
+                    result['msg'] = msg
+
+                return result
+
+        return dict(changed=False, msg='all assertions passed')
+

v2/ansible/plugins/action/copy.py

@@ -0,0 +1,384 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import base64
+import json
+import os
+import pipes
+import stat
+import tempfile
+
+from ansible import constants as C
+from ansible.plugins.action import ActionBase
+from ansible.utils.boolean import boolean
+from ansible.utils.hashing import checksum
+
+## fixes https://github.com/ansible/ansible/issues/3518
+# http://mypy.pythonblogs.com/12_mypy/archive/1253_workaround_for_python_bug_ascii_codec_cant_encode_character_uxa0_in_position_111_ordinal_not_in_range128.html
+
+import sys
+reload(sys)
+sys.setdefaultencoding("utf8")
+
+
+class ActionModule(ActionBase):
+
+    def run(self, tmp=None, task_vars=dict()):
+        ''' handler for file transfer operations '''
+
+        source  = self._task.args.get('src', None)
+        content = self._task.args.get('content', None)
+        dest    = self._task.args.get('dest', None)
+        raw     = boolean(self._task.args.get('raw', 'no'))
+        force   = boolean(self._task.args.get('force', 'yes'))
+
+        # content with newlines is going to be escaped to safely load in yaml
+        # now we need to unescape it so that the newlines are evaluated properly
+        # when writing the file to disk
+        if content:
+            if isinstance(content, unicode):
+                try:
+                    content = content.decode('unicode-escape')
+                except UnicodeDecodeError:
+                    pass
+
+        # FIXME: first available file needs to be reworked somehow...
+        #if (source is None and content is None and not 'first_available_file' in inject) or dest is None:
+        #    result=dict(failed=True, msg="src (or content) and dest are required")
+        #    return ReturnData(conn=conn, result=result)
+        #elif (source is not None or 'first_available_file' in inject) and content is not None:
+        #    result=dict(failed=True, msg="src and content are mutually exclusive")
+        #    return ReturnData(conn=conn, result=result)
+
+        # Check if the source ends with a "/"
+        source_trailing_slash = False
+        if source:
+            source_trailing_slash = source.endswith("/")
+
+        # Define content_tempfile in case we set it after finding content populated.
+        content_tempfile = None
+
+        # If content is defined make a temp file and write the content into it.
+        if content is not None:
+            try:
+                # If content comes to us as a dict it should be decoded json.
+                # We need to encode it back into a string to write it out.
+                if type(content) is dict:
+                    content_tempfile = self._create_content_tempfile(json.dumps(content))
+                else:
+                    content_tempfile = self._create_content_tempfile(content)
+                source = content_tempfile
+            except Exception, err:
+                result = dict(failed=True, msg="could not write content temp file: %s" % err)
+                return ReturnData(conn=conn, result=result)
+        ###############################################################################################
+        # FIXME: first_available_file needs to be reworked?
+        ###############################################################################################
+        # if we have first_available_file in our vars
+        # look up the files and use the first one we find as src
+        #elif 'first_available_file' in inject:
+        #    found = False
+        #    for fn in inject.get('first_available_file'):
+        #        fn_orig = fn
+        #        fnt = template.template(self.runner.basedir, fn, inject)
+        #        fnd = utils.path_dwim(self.runner.basedir, fnt)
+        #        if not os.path.exists(fnd) and '_original_file' in inject:
+        #            fnd = utils.path_dwim_relative(inject['_original_file'], 'files', fnt, self.runner.basedir, check=False)
+        #        if os.path.exists(fnd):
+        #            source = fnd
+        #            found = True
+        #            break
+        #    if not found:
+        #        results = dict(failed=True, msg="could not find src in first_available_file list")
+        #        return ReturnData(conn=conn, result=results)
+        ###############################################################################################
+        else:
+            # FIXME: templating needs to be worked out still
+            #source = template.template(self.runner.basedir, source, inject)
+            # FIXME: original_file stuff needs to be reworked - most likely
+            #        simply checking to see if the task has a role and using
+            #        using the role path as the dwim target and basedir would work
+            #if '_original_file' in inject:
+            #    source = utils.path_dwim_relative(inject['_original_file'], 'files', source, self.runner.basedir)
+            #else:
+            #    source = utils.path_dwim(self.runner.basedir, source)
+            source = self._loader.path_dwim(source)
+
+        # A list of source file tuples (full_path, relative_path) which will try to copy to the destination
+        source_files = []
+
+        # If source is a directory populate our list else source is a file and translate it to a tuple.
+        if os.path.isdir(source):
+            # Get the amount of spaces to remove to get the relative path.
+            if source_trailing_slash:
+                sz = len(source) + 1
+            else:
+                sz = len(source.rsplit('/', 1)[0]) + 1
+
+            # Walk the directory and append the file tuples to source_files.
+            for base_path, sub_folders, files in os.walk(source):
+                for file in files:
+                    full_path = os.path.join(base_path, file)
+                    rel_path = full_path[sz:]
+                    source_files.append((full_path, rel_path))
+
+            # If it's recursive copy, destination is always a dir,
+            # explicitly mark it so (note - copy module relies on this).
+            if not self._shell.path_has_trailing_slash(dest):
+                dest = self._shell.join_path(dest, '')
+        else:
+            source_files.append((source, os.path.basename(source)))
+
+        changed = False
+        diffs = []
+        module_result = {"changed": False}
+
+        # A register for if we executed a module.
+        # Used to cut down on command calls when not recursive.
+        module_executed = False
+
+        # Tell _execute_module to delete the file if there is one file.
+        delete_remote_tmp = (len(source_files) == 1)
+
+        # If this is a recursive action create a tmp path that we can share as the _exec_module create is too late.
+        if not delete_remote_tmp:
+            if tmp is None or "-tmp-" not in tmp:
+                tmp = self._make_tmp_path()
+
+        # expand any user home dir specifier
+        dest = self._remote_expand_user(dest, tmp)
+
+        for source_full, source_rel in source_files:
+            # Generate a hash of the local file.
+            local_checksum = checksum(source_full)
+
+            # If local_checksum is not defined we can't find the file so we should fail out.
+            if local_checksum is None:
+                return dict(failed=True, msg="could not find src=%s" % source_full)
+
+            # This is kind of optimization - if user told us destination is
+            # dir, do path manipulation right away, otherwise we still check
+            # for dest being a dir via remote call below.
+            if self._shell.path_has_trailing_slash(dest):
+                dest_file = self._shell.join_path(dest, source_rel)
+            else:
+                dest_file = self._shell.join_path(dest)
+
+            # Attempt to get the remote checksum
+            remote_checksum = self._remote_checksum(tmp, dest_file)
+
+            if remote_checksum == '3':
+                # The remote_checksum was executed on a directory.
+                if content is not None:
+                    # If source was defined as content remove the temporary file and fail out.
+                    self._remove_tempfile_if_content_defined(content, content_tempfile)
+                    return dict(failed=True, msg="can not use content with a dir as dest")
+                else:
+                    # Append the relative source location to the destination and retry remote_checksum
+                    dest_file = self._shell.join_path(dest, source_rel)
+                    remote_checksum = self._remote_checksum(tmp, dest_file)
+
+            if remote_checksum != '1' and not force:
+                # remote_file does not exist so continue to next iteration.
+                continue
+
+            if local_checksum != remote_checksum:
+                # The checksums don't match and we will change or error out.
+                changed = True
+
+                # Create a tmp path if missing only if this is not recursive.
+                # If this is recursive we already have a tmp path.
+                if delete_remote_tmp:
+                    if tmp is None or "-tmp-" not in tmp:
+                        tmp = self._make_tmp_path()
+
+                # FIXME: runner shouldn't have the diff option there
+                #if self.runner.diff and not raw:
+                #    diff = self._get_diff_data(tmp, dest_file, source_full)
+                #else:
+                #    diff = {}
+                diff = {}
+
+                # FIXME: noop stuff
+                #if self.runner.noop_on_check(inject):
+                #    self._remove_tempfile_if_content_defined(content, content_tempfile)
+                #    diffs.append(diff)
+                #    changed = True
+                #    module_result = dict(changed=True)
+                #    continue
+
+                # Define a remote directory that we will copy the file to.
+                tmp_src = tmp + 'source'
+
+                if not raw:
+                    self._connection.put_file(source_full, tmp_src)
+                else:
+                    self._connection.put_file(source_full, dest_file)
+
+                # We have copied the file remotely and no longer require our content_tempfile
+                self._remove_tempfile_if_content_defined(content, content_tempfile)
+
+                # fix file permissions when the copy is done as a different user
+                if (self._connection_info.sudo and self._connection_info.sudo_user != 'root' or self._connection_info.su and self._connection_info.su_user != 'root') and not raw:
+                    self._remote_chmod('a+r', tmp_src, tmp)
+
+                if raw:
+                    # Continue to next iteration if raw is defined.
+                    continue
+
+                # Run the copy module
+
+                # src and dest here come after original and override them
+                # we pass dest only to make sure it includes trailing slash in case of recursive copy
+                new_module_args = self._task.args.copy()
+                new_module_args.update(
+                    dict(
+                        src=tmp_src,
+                        dest=dest,
+                        original_basename=source_rel,
+                    )
+                )
+
+                # FIXME: checkmode and no_log stuff
+                #if self.runner.noop_on_check(inject):
+                #    new_module_args['CHECKMODE'] = True
+                #if self.runner.no_log:
+                #    new_module_args['NO_LOG'] = True
+
+                module_return = self._execute_module(module_name='copy', module_args=new_module_args, tmp=tmp, delete_remote_tmp=delete_remote_tmp)
+                module_executed = True
+
+            else:
+                # no need to transfer the file, already correct hash, but still need to call
+                # the file module in case we want to change attributes
+                self._remove_tempfile_if_content_defined(content, content_tempfile)
+
+                if raw:
+                    # Continue to next iteration if raw is defined.
+                    # self._remove_tmp_path(tmp)
+                    continue
+
+                # Build temporary module_args.
+                new_module_args = self._task.args.copy()
+                new_module_args.update(
+                    dict(
+                        src=source_rel,
+                        dest=dest,
+                        original_basename=source_rel
+                    )
+                )
+                # FIXME: checkmode and no_log stuff
+                #if self.runner.noop_on_check(inject):
+                #    new_module_args['CHECKMODE'] = True
+                #if self.runner.no_log:
+                #    new_module_args['NO_LOG'] = True
+
+                # Execute the file module.
+                module_return = self._execute_module(module_name='file', module_args=new_module_args, tmp=tmp, delete_remote_tmp=delete_remote_tmp)
+                module_executed = True
+
+            if not module_return.get('checksum'):
+                module_return['checksum'] = local_checksum
+            if module_return.get('failed') == True:
+                return module_return
+            if module_return.get('changed') == True:
+                changed = True
+
+        # Delete tmp path if we were recursive or if we did not execute a module.
+        if (not C.DEFAULT_KEEP_REMOTE_FILES and not delete_remote_tmp) \
+            or (not C.DEFAULT_KEEP_REMOTE_FILES and delete_remote_tmp and not module_executed):
+            self._remove_tmp_path(tmp)
+
+        # the file module returns the file path as 'path', but 
+        # the copy module uses 'dest', so add it if it's not there
+        if 'path' in module_return and 'dest' not in module_return:
+            module_return['dest'] = module_return['path']
+
+        # TODO: Support detailed status/diff for multiple files
+        if len(source_files) == 1:
+            result = module_return
+        else:
+            result = dict(dest=dest, src=source, changed=changed)
+
+        # FIXME: move diffs into the result?
+        #if len(diffs) == 1:
+        #    return ReturnData(conn=conn, result=result, diff=diffs[0])
+        #else:
+        #    return ReturnData(conn=conn, result=result)
+
+        return result
+
+    def _create_content_tempfile(self, content):
+        ''' Create a tempfile containing defined content '''
+        fd, content_tempfile = tempfile.mkstemp()
+        f = os.fdopen(fd, 'w')
+        try:
+            f.write(content)
+        except Exception, err:
+            os.remove(content_tempfile)
+            raise Exception(err)
+        finally:
+            f.close()
+        return content_tempfile
+
+    def _get_diff_data(self, tmp, destination, source):
+        peek_result = self._execute_module(module_name='file', module_args=dict(path=destination, diff_peek=True), persist_files=True)
+        if 'failed' in peek_result and peek_result['failed'] or peek_result.get('rc', 0) != 0:
+            return {}
+
+        diff = {}
+        if peek_result['state'] == 'absent':
+            diff['before'] = ''
+        elif peek_result['appears_binary']:
+            diff['dst_binary'] = 1
+        # FIXME: this should not be in utils..
+        #elif peek_result['size'] > utils.MAX_FILE_SIZE_FOR_DIFF:
+        #    diff['dst_larger'] = utils.MAX_FILE_SIZE_FOR_DIFF
+        else:
+            dest_result = self._execute_module(module_name='slurp', module_args=dict(path=destination), tmp=tmp, persist_files=True)
+            if 'content' in dest_result:
+                dest_contents = dest_result['content']
+                if dest_result['encoding'] == 'base64':
+                    dest_contents = base64.b64decode(dest_contents)
+                else:
+                    raise Exception("unknown encoding, failed: %s" % dest_result)
+                diff['before_header'] = destination
+                diff['before'] = dest_contents
+
+        src = open(source)
+        src_contents = src.read(8192)
+        st = os.stat(source)
+        if "\x00" in src_contents:
+            diff['src_binary'] = 1
+        # FIXME: this should not be in utils
+        #elif st[stat.ST_SIZE] > utils.MAX_FILE_SIZE_FOR_DIFF:
+        #    diff['src_larger'] = utils.MAX_FILE_SIZE_FOR_DIFF
+        else:
+            src.seek(0)
+            diff['after_header'] = source
+            diff['after'] = src.read()
+
+        return diff
+
+    def _remove_tempfile_if_content_defined(self, content, content_tempfile):
+        if content is not None:
+            os.remove(content_tempfile)
+

v2/ansible/plugins/action/debug.py

@@ -0,0 +1,46 @@
+# Copyright 2012, Dag Wieers <dag@wieers.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from ansible.plugins.action import ActionBase
+from ansible.utils.boolean import boolean
+from ansible.template import Templar
+
+class ActionModule(ActionBase):
+    ''' Print statements during execution '''
+
+    TRANSFERS_FILES = False
+
+    def run(self, tmp=None, task_vars=dict()):
+
+        if 'msg' in self._task.args:
+            if 'fail' in self._task.args and boolean(self._task.args['fail']):
+                result = dict(failed=True, msg=self._task.args['msg'])
+            else:
+                result = dict(msg=self._task.args['msg'])
+        # FIXME: move the LOOKUP_REGEX somewhere else
+        elif 'var' in self._task.args: # and not utils.LOOKUP_REGEX.search(self._task.args['var']):
+            templar = Templar(variables=task_vars)
+            results = templar.template(self._task.args['var'], convert_bare=True)
+            result = dict()
+            result[self._task.args['var']] = results
+        else:
+            result = dict(msg='here we are')
+
+        # force flag to make debug output module always verbose
+        result['verbose_always'] = True
+
+        return result

v2/ansible/plugins/action/include_vars.py

@@ -0,0 +1,48 @@
+# (c) 2013-2014, Benno Joy <benno@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from types import NoneType
+
+from ansible.errors import AnsibleError
+from ansible.parsing import DataLoader
+from ansible.plugins.action import ActionBase
+
+class ActionModule(ActionBase):
+
+    TRANSFERS_FILES = False
+
+    def run(self, tmp=None, task_vars=dict()):
+
+        source = self._task.args.get('_raw_params')
+
+        if self._task._role:
+            source = self._loader.path_dwim_relative(self._task._role.get('_role_path',''), 'vars', source)
+        else:
+            source = self._loader.path_dwim(source)
+
+        if os.path.exists(source):
+            data = self._loader.load_from_file(source)
+            if data is None:
+                data = {}
+            if not isinstance(data, dict):
+                raise AnsibleError("%s must be stored as a dictionary/hash" % source)
+            return dict(ansible_facts=data)
+        else:
+            return dict(failed=True, msg="Source file not found.", file=source)
+

v2/ansible/plugins/action/normal.py

@@ -0,0 +1,40 @@
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from ansible.plugins.action import ActionBase
+
+class ActionModule(ActionBase):
+
+    def run(self, tmp=None, task_vars=dict()):
+
+        # FIXME: a lot of this should pretty much go away with module
+        #        args being stored within the task being run itself
+
+        #if self.runner.noop_on_check(inject):
+        #    if module_name in [ 'shell', 'command' ]:
+        #        return ReturnData(conn=conn, comm_ok=True, result=dict(skipped=True, msg='check mode not supported for %s' % module_name))
+        #    # else let the module parsing code decide, though this will only be allowed for AnsibleModuleCommon using
+        #    # python modules for now
+        #    module_args += " CHECKMODE=True"
+
+        #if self.runner.no_log:
+        #    module_args += " NO_LOG=True"
+
+        #vv("REMOTE_MODULE %s %s" % (module_name, module_args), host=conn.host)
+        return self._execute_module(tmp)
+
+

v2/ansible/plugins/action/set_fact.py

@@ -1,4 +1,4 @@
-# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+# Copyright 2013 Dag Wieers <dag@wieers.com>
 #
 # This file is part of Ansible
 #
@@ -15,19 +15,12 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
-from ansible.utils import template
-import ansible.utils as utils
+from ansible.errors import AnsibleError
+from ansible.plugins.action import ActionBase
 
-class LookupModule(object):
+class ActionModule(ActionBase):
 
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
+    TRANSFERS_FILES = False
 
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
-
-        ret = []
-        for term in terms:
-            ret.append(template.template_from_file(self.basedir, term, inject))
-        return ret
+    def run(self, tmp=None, task_vars=dict()):
+        return dict(changed=True, ansible_facts=self._task.args)

v2/ansible/plugins/callback/__init__.py

@@ -19,3 +19,86 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+from ansible.utils.display import Display
+
+__all__ = ["CallbackBase"]
+
+class CallbackBase:
+
+    '''
+    This is a base ansible callback class that does nothing. New callbacks should
+    use this class as a base and override any callback methods they wish to execute
+    custom actions.
+    '''
+
+    # FIXME: the list of functions here needs to be updated once we have
+    #        finalized the list of callback methods used in the default callback
+
+    def __init__(self):
+        self._display = Display()
+
+    def set_connection_info(self, conn_info):
+        # FIXME: this is a temporary hack, as the connection info object
+        #        should be created early and passed down through objects
+        self._display._verbosity = conn_info.verbosity
+
+    def on_any(self, *args, **kwargs):
+        pass
+
+    def runner_on_failed(self, host, res, ignore_errors=False):
+        pass
+
+    def runner_on_ok(self, host, res):
+        pass
+
+    def runner_on_skipped(self, host, item=None):
+        pass
+
+    def runner_on_unreachable(self, host, res):
+        pass
+
+    def runner_on_no_hosts(self):
+        pass
+
+    def runner_on_async_poll(self, host, res, jid, clock):
+        pass
+
+    def runner_on_async_ok(self, host, res, jid):
+        pass
+
+    def runner_on_async_failed(self, host, res, jid):
+        pass
+
+    def playbook_on_start(self):
+        pass
+
+    def playbook_on_notify(self, host, handler):
+        pass
+
+    def playbook_on_no_hosts_matched(self):
+        pass
+
+    def playbook_on_no_hosts_remaining(self):
+        pass
+
+    def playbook_on_task_start(self, name, is_conditional):
+        pass
+
+    def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None):
+        pass
+
+    def playbook_on_setup(self):
+        pass
+
+    def playbook_on_import_for_host(self, host, imported_file):
+        pass
+
+    def playbook_on_not_import_for_host(self, host, missing_file):
+        pass
+
+    def playbook_on_play_start(self, name):
+        pass
+
+    def playbook_on_stats(self, stats):
+        pass
+

v2/ansible/plugins/callback/default.py

@@ -0,0 +1,120 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+from ansible.plugins.callback import CallbackBase
+
+class CallbackModule(CallbackBase):
+
+    '''
+    This is the default callback interface, which simply prints messages
+    to stdout when new callback events are received.
+    '''
+
+    def _print_banner(self, msg, color=None):
+        '''
+        Prints a header-looking line with stars taking up to 80 columns
+        of width (3 columns, minimum)
+        '''
+        msg = msg.strip()
+        star_len = (80 - len(msg))
+        if star_len < 0:
+            star_len = 3
+        stars = "*" * star_len
+        self._display.display("\n%s %s" % (msg, stars), color=color)
+
+    def on_any(self, *args, **kwargs):
+        pass
+
+    def runner_on_failed(self, task, result, ignore_errors=False):
+        self._display.display("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), result._result), color='red')
+
+    def runner_on_ok(self, task, result):
+        msg = "ok: [%s]" % result._host.get_name()
+        if self._display._verbosity > 0 or 'verbose_always' in result._result:
+            if 'verbose_always' in result._result:
+                del result._result['verbose_always']
+            msg += " => %s" % result._result
+        self._display.display(msg, color='green')
+
+    def runner_on_skipped(self, task, result):
+        msg = "SKIPPED: [%s]" % result._host.get_name()
+        if self._display._verbosity > 0 or 'verbose_always' in result._result:
+            if 'verbose_always' in result._result:
+                del result._result['verbose_always']
+            msg += " => %s" % result._result
+        self._display.display(msg)
+
+    def runner_on_unreachable(self, task, result):
+        self._display.display("fatal: [%s]: UNREACHABLE! => %s" % (result._host.get_name(), result._result), color='red')
+
+    def runner_on_no_hosts(self, task):
+        pass
+
+    def runner_on_async_poll(self, host, res, jid, clock):
+        pass
+
+    def runner_on_async_ok(self, host, res, jid):
+        pass
+
+    def runner_on_async_failed(self, host, res, jid):
+        pass
+
+    def playbook_on_start(self):
+        pass
+
+    def playbook_on_notify(self, host, handler):
+        pass
+
+    def playbook_on_no_hosts_matched(self):
+        pass
+
+    def playbook_on_no_hosts_remaining(self):
+        self._print_banner("NO MORE HOSTS LEFT")
+
+    def playbook_on_task_start(self, name, is_conditional):
+        self._print_banner("TASK [%s]" % name.strip())
+
+    def playbook_on_cleanup_task_start(self, name):
+        self._print_banner("CLEANUP TASK [%s]" % name.strip())
+
+    def playbook_on_handler_task_start(self, name):
+        self._print_banner("RUNNING HANDLER [%s]" % name.strip())
+
+    def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None):
+        pass
+
+    def playbook_on_setup(self):
+        pass
+
+    def playbook_on_import_for_host(self, host, imported_file):
+        pass
+
+    def playbook_on_not_import_for_host(self, host, missing_file):
+        pass
+
+    def playbook_on_play_start(self, name):
+        self._print_banner("PLAY [%s]" % name.strip())
+
+    def playbook_on_stats(self, stats):
+        pass
+

v2/ansible/plugins/callback/minimal.py

@@ -0,0 +1,111 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+from ansible.plugins.callback import CallbackBase
+
+
+class CallbackModule(CallbackBase):
+
+    '''
+    This is the default callback interface, which simply prints messages
+    to stdout when new callback events are received.
+    '''
+
+    def _print_banner(self, msg):
+        '''
+        Prints a header-looking line with stars taking up to 80 columns
+        of width (3 columns, minimum)
+        '''
+        msg = msg.strip()
+        star_len = (80 - len(msg))
+        if star_len < 0:
+            star_len = 3
+        stars = "*" * star_len
+        self._display.display("\n%s %s\n" % (msg, stars))
+
+    def on_any(self, *args, **kwargs):
+        pass
+
+    def runner_on_failed(self, task, result, ignore_errors=False):
+        self._display.display("%s | FAILED! => %s" % (result._host.get_name(), result._result), color='red')
+
+    def runner_on_ok(self, task, result):
+        self._display.display("%s | SUCCESS => %s" % (result._host.get_name(), json.dumps(result._result, indent=4)), color='green')
+
+    def runner_on_skipped(self, task, result):
+        pass
+
+    def runner_on_unreachable(self, task, result):
+        self._display.display("%s | UNREACHABLE!" % result._host.get_name(), color='yellow')
+
+    def runner_on_no_hosts(self, task):
+        pass
+
+    def runner_on_async_poll(self, host, res, jid, clock):
+        pass
+
+    def runner_on_async_ok(self, host, res, jid):
+        pass
+
+    def runner_on_async_failed(self, host, res, jid):
+        pass
+
+    def playbook_on_start(self):
+        pass
+
+    def playbook_on_notify(self, host, handler):
+        pass
+
+    def playbook_on_no_hosts_matched(self):
+        pass
+
+    def playbook_on_no_hosts_remaining(self):
+        pass
+
+    def playbook_on_task_start(self, name, is_conditional):
+        pass
+
+    def playbook_on_cleanup_task_start(self, name):
+        pass
+
+    def playbook_on_handler_task_start(self, name):
+        pass
+
+    def playbook_on_vars_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None):
+        pass
+
+    def playbook_on_setup(self):
+        pass
+
+    def playbook_on_import_for_host(self, host, imported_file):
+        pass
+
+    def playbook_on_not_import_for_host(self, host, missing_file):
+        pass
+
+    def playbook_on_play_start(self, name):
+        pass
+
+    def playbook_on_stats(self, stats):
+        pass
+

v2/ansible/plugins/connections/__init__.py

@@ -19,3 +19,24 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+from ansible import constants as C
+
+# FIXME: this object should be created upfront and passed through
+#        the entire chain of calls to here, as there are other things
+#        which may want to output display/logs too
+from ansible.utils.display import Display
+
+__all__ = ['ConnectionBase']
+
+
+class ConnectionBase:
+    '''
+    A base class for connections to contain common code.
+    '''
+
+    def __init__(self, host, connection_info, *args, **kwargs):
+        self._host = host
+        self._connection_info = connection_info
+        self._has_pipelining = False
+        self._display = Display(connection_info)
+

v2/ansible/plugins/connections/accelerate.py

@@ -0,0 +1,371 @@
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import os
+import base64
+import socket
+import struct
+import time
+from ansible.callbacks import vvv, vvvv
+from ansible.errors import AnsibleError, AnsibleFileNotFound
+from ansible.runner.connection_plugins.ssh import Connection as SSHConnection
+from ansible.runner.connection_plugins.paramiko_ssh import Connection as ParamikoConnection
+from ansible import utils
+from ansible import constants
+
+# the chunk size to read and send, assuming mtu 1500 and
+# leaving room for base64 (+33%) encoding and header (8 bytes)
+# ((1400-8)/4)*3) = 1044
+# which leaves room for the TCP/IP header. We set this to a 
+# multiple of the value to speed up file reads.
+CHUNK_SIZE=1044*20
+
+class Connection(object):
+    ''' raw socket accelerated connection '''
+
+    def __init__(self, runner, host, port, user, password, private_key_file, *args, **kwargs):
+
+        self.runner = runner
+        self.host = host
+        self.context = None
+        self.conn = None
+        self.user = user
+        self.key = utils.key_for_hostname(host)
+        self.port = port[0]
+        self.accport = port[1]
+        self.is_connected = False
+        self.has_pipelining = False
+
+        if not self.port:
+            self.port = constants.DEFAULT_REMOTE_PORT
+        elif not isinstance(self.port, int):
+            self.port = int(self.port)
+
+        if not self.accport:
+            self.accport = constants.ACCELERATE_PORT
+        elif not isinstance(self.accport, int):
+            self.accport = int(self.accport)
+
+        if self.runner.original_transport == "paramiko":
+            self.ssh = ParamikoConnection(
+                runner=self.runner,
+                host=self.host,
+                port=self.port,
+                user=self.user,
+                password=password,
+                private_key_file=private_key_file
+            )
+        else:
+            self.ssh = SSHConnection(
+                runner=self.runner,
+                host=self.host,
+                port=self.port,
+                user=self.user,
+                password=password,
+                private_key_file=private_key_file
+            )
+
+        if not getattr(self.ssh, 'shell', None):
+            self.ssh.shell = utils.plugins.shell_loader.get('sh')
+
+        # attempt to work around shared-memory funness
+        if getattr(self.runner, 'aes_keys', None):
+            utils.AES_KEYS = self.runner.aes_keys
+
+    def _execute_accelerate_module(self):
+        args = "password=%s port=%s minutes=%d debug=%d ipv6=%s" % (
+            base64.b64encode(self.key.__str__()), 
+            str(self.accport), 
+            constants.ACCELERATE_DAEMON_TIMEOUT, 
+            int(utils.VERBOSITY), 
+            self.runner.accelerate_ipv6,
+        )
+        if constants.ACCELERATE_MULTI_KEY:
+            args += " multi_key=yes"
+        inject = dict(password=self.key)
+        if getattr(self.runner, 'accelerate_inventory_host', False):
+            inject = utils.combine_vars(inject, self.runner.inventory.get_variables(self.runner.accelerate_inventory_host))
+        else:
+            inject = utils.combine_vars(inject, self.runner.inventory.get_variables(self.host))
+        vvvv("attempting to start up the accelerate daemon...")
+        self.ssh.connect()
+        tmp_path = self.runner._make_tmp_path(self.ssh)
+        return self.runner._execute_module(self.ssh, tmp_path, 'accelerate', args, inject=inject)
+
+    def connect(self, allow_ssh=True):
+        ''' activates the connection object '''
+
+        try:
+            if not self.is_connected:
+                wrong_user = False
+                tries = 3
+                self.conn = socket.socket()
+                self.conn.settimeout(constants.ACCELERATE_CONNECT_TIMEOUT)
+                vvvv("attempting connection to %s via the accelerated port %d" % (self.host,self.accport))
+                while tries > 0:
+                    try:
+                        self.conn.connect((self.host,self.accport))
+                        break
+                    except socket.error:
+                        vvvv("connection to %s failed, retrying..." % self.host)
+                        time.sleep(0.1)
+                        tries -= 1
+                if tries == 0:
+                    vvv("Could not connect via the accelerated connection, exceeded # of tries")
+                    raise AnsibleError("FAILED")
+                elif wrong_user:
+                    vvv("Restarting daemon with a different remote_user")
+                    raise AnsibleError("WRONG_USER")
+
+                self.conn.settimeout(constants.ACCELERATE_TIMEOUT)
+                if not self.validate_user():
+                    # the accelerated daemon was started with a 
+                    # different remote_user. The above command
+                    # should have caused the accelerate daemon to
+                    # shutdown, so we'll reconnect.
+                    wrong_user = True
+
+        except AnsibleError, e:
+            if allow_ssh:
+                if "WRONG_USER" in e:
+                    vvv("Switching users, waiting for the daemon on %s to shutdown completely..." % self.host)
+                    time.sleep(5)
+                vvv("Falling back to ssh to startup accelerated mode")
+                res = self._execute_accelerate_module()
+                if not res.is_successful():
+                    raise AnsibleError("Failed to launch the accelerated daemon on %s (reason: %s)" % (self.host,res.result.get('msg')))
+                return self.connect(allow_ssh=False)
+            else:
+                raise AnsibleError("Failed to connect to %s:%s" % (self.host,self.accport))
+        self.is_connected = True
+        return self
+
+    def send_data(self, data):
+        packed_len = struct.pack('!Q',len(data))
+        return self.conn.sendall(packed_len + data)
+
+    def recv_data(self):
+        header_len = 8 # size of a packed unsigned long long
+        data = b""
+        try:
+            vvvv("%s: in recv_data(), waiting for the header" % self.host)
+            while len(data) < header_len:
+                d = self.conn.recv(header_len - len(data))
+                if not d:
+                    vvvv("%s: received nothing, bailing out" % self.host)
+                    return None
+                data += d
+            vvvv("%s: got the header, unpacking" % self.host)
+            data_len = struct.unpack('!Q',data[:header_len])[0]
+            data = data[header_len:]
+            vvvv("%s: data received so far (expecting %d): %d" % (self.host,data_len,len(data)))
+            while len(data) < data_len:
+                d = self.conn.recv(data_len - len(data))
+                if not d:
+                    vvvv("%s: received nothing, bailing out" % self.host)
+                    return None
+                vvvv("%s: received %d bytes" % (self.host, len(d)))
+                data += d
+            vvvv("%s: received all of the data, returning" % self.host)
+            return data
+        except socket.timeout:
+            raise AnsibleError("timed out while waiting to receive data")
+
+    def validate_user(self):
+        '''
+        Checks the remote uid of the accelerated daemon vs. the 
+        one specified for this play and will cause the accel 
+        daemon to exit if they don't match
+        '''
+
+        vvvv("%s: sending request for validate_user" % self.host)
+        data = dict(
+            mode='validate_user',
+            username=self.user,
+        )
+        data = utils.jsonify(data)
+        data = utils.encrypt(self.key, data)
+        if self.send_data(data):
+            raise AnsibleError("Failed to send command to %s" % self.host)
+
+        vvvv("%s: waiting for validate_user response" % self.host)
+        while True:
+            # we loop here while waiting for the response, because a
+            # long running command may cause us to receive keepalive packets
+            # ({"pong":"true"}) rather than the response we want.
+            response = self.recv_data()
+            if not response:
+                raise AnsibleError("Failed to get a response from %s" % self.host)
+            response = utils.decrypt(self.key, response)
+            response = utils.parse_json(response)
+            if "pong" in response:
+                # it's a keepalive, go back to waiting
+                vvvv("%s: received a keepalive packet" % self.host)
+                continue
+            else:
+                vvvv("%s: received the validate_user response: %s" % (self.host, response))
+                break
+
+        if response.get('failed'):
+            return False
+        else:
+            return response.get('rc') == 0
+
+    def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, executable='/bin/sh', in_data=None, su=None, su_user=None):
+        ''' run a command on the remote host '''
+
+        if su or su_user:
+            raise AnsibleError("Internal Error: this module does not support running commands via su")
+
+        if in_data:
+            raise AnsibleError("Internal Error: this module does not support optimized module pipelining")
+
+        if executable == "":
+            executable = constants.DEFAULT_EXECUTABLE
+
+        if self.runner.sudo and sudoable and sudo_user:
+            cmd, prompt, success_key = utils.make_sudo_cmd(self.runner.sudo_exe, sudo_user, executable, cmd)
+
+        vvv("EXEC COMMAND %s" % cmd)
+
+        data = dict(
+            mode='command',
+            cmd=cmd,
+            tmp_path=tmp_path,
+            executable=executable,
+        )
+        data = utils.jsonify(data)
+        data = utils.encrypt(self.key, data)
+        if self.send_data(data):
+            raise AnsibleError("Failed to send command to %s" % self.host)
+        
+        while True:
+            # we loop here while waiting for the response, because a 
+            # long running command may cause us to receive keepalive packets
+            # ({"pong":"true"}) rather than the response we want. 
+            response = self.recv_data()
+            if not response:
+                raise AnsibleError("Failed to get a response from %s" % self.host)
+            response = utils.decrypt(self.key, response)
+            response = utils.parse_json(response)
+            if "pong" in response:
+                # it's a keepalive, go back to waiting
+                vvvv("%s: received a keepalive packet" % self.host)
+                continue
+            else:
+                vvvv("%s: received the response" % self.host)
+                break
+
+        return (response.get('rc',None), '', response.get('stdout',''), response.get('stderr',''))
+
+    def put_file(self, in_path, out_path):
+
+        ''' transfer a file from local to remote '''
+        vvv("PUT %s TO %s" % (in_path, out_path), host=self.host)
+
+        if not os.path.exists(in_path):
+            raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+
+        fd = file(in_path, 'rb')
+        fstat = os.stat(in_path)
+        try:
+            vvv("PUT file is %d bytes" % fstat.st_size)
+            last = False
+            while fd.tell() <= fstat.st_size and not last:
+                vvvv("file position currently %ld, file size is %ld" % (fd.tell(), fstat.st_size))
+                data = fd.read(CHUNK_SIZE)
+                if fd.tell() >= fstat.st_size:
+                    last = True
+                data = dict(mode='put', data=base64.b64encode(data), out_path=out_path, last=last)
+                if self.runner.sudo:
+                    data['user'] = self.runner.sudo_user
+                data = utils.jsonify(data)
+                data = utils.encrypt(self.key, data)
+
+                if self.send_data(data):
+                    raise AnsibleError("failed to send the file to %s" % self.host)
+
+                response = self.recv_data()
+                if not response:
+                    raise AnsibleError("Failed to get a response from %s" % self.host)
+                response = utils.decrypt(self.key, response)
+                response = utils.parse_json(response)
+
+                if response.get('failed',False):
+                    raise AnsibleError("failed to put the file in the requested location")
+        finally:
+            fd.close()
+            vvvv("waiting for final response after PUT")
+            response = self.recv_data()
+            if not response:
+                raise AnsibleError("Failed to get a response from %s" % self.host)
+            response = utils.decrypt(self.key, response)
+            response = utils.parse_json(response)
+
+            if response.get('failed',False):
+                raise AnsibleError("failed to put the file in the requested location")
+
+    def fetch_file(self, in_path, out_path):
+        ''' save a remote file to the specified path '''
+        vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host)
+
+        data = dict(mode='fetch', in_path=in_path)
+        data = utils.jsonify(data)
+        data = utils.encrypt(self.key, data)
+        if self.send_data(data):
+            raise AnsibleError("failed to initiate the file fetch with %s" % self.host)
+
+        fh = open(out_path, "w")
+        try:
+            bytes = 0
+            while True:
+                response = self.recv_data()
+                if not response:
+                    raise AnsibleError("Failed to get a response from %s" % self.host)
+                response = utils.decrypt(self.key, response)
+                response = utils.parse_json(response)
+                if response.get('failed', False):
+                    raise AnsibleError("Error during file fetch, aborting")
+                out = base64.b64decode(response['data'])
+                fh.write(out)
+                bytes += len(out)
+                # send an empty response back to signify we 
+                # received the last chunk without errors
+                data = utils.jsonify(dict())
+                data = utils.encrypt(self.key, data)
+                if self.send_data(data):
+                    raise AnsibleError("failed to send ack during file fetch")
+                if response.get('last', False):
+                    break
+        finally:
+            # we don't currently care about this final response,
+            # we just receive it and drop it. It may be used at some
+            # point in the future or we may just have the put/fetch
+            # operations not send back a final response at all
+            response = self.recv_data()
+            vvv("FETCH wrote %d bytes to %s" % (bytes, out_path))
+            fh.close()
+
+    def close(self):
+        ''' terminate the connection '''
+        # Be a good citizen
+        try:
+            self.conn.close()
+        except:
+            pass
+

v2/ansible/plugins/connections/chroot.py

@@ -0,0 +1,130 @@
+# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2013, Maykel Moya <mmoya@speedyrails.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import distutils.spawn
+import traceback
+import os
+import shutil
+import subprocess
+from ansible import errors
+from ansible import utils
+from ansible.callbacks import vvv
+
+class Connection(object):
+    ''' Local chroot based connections '''
+
+    def __init__(self, runner, host, port, *args, **kwargs):
+        self.chroot = host
+        self.has_pipelining = False
+
+        if os.geteuid() != 0:
+            raise errors.AnsibleError("chroot connection requires running as root")
+
+        # we're running as root on the local system so do some
+        # trivial checks for ensuring 'host' is actually a chroot'able dir
+        if not os.path.isdir(self.chroot):
+            raise errors.AnsibleError("%s is not a directory" % self.chroot)
+
+        chrootsh = os.path.join(self.chroot, 'bin/sh')
+        if not utils.is_executable(chrootsh):
+            raise errors.AnsibleError("%s does not look like a chrootable dir (/bin/sh missing)" % self.chroot)
+
+        self.chroot_cmd = distutils.spawn.find_executable('chroot')
+        if not self.chroot_cmd:
+            raise errors.AnsibleError("chroot command not found in PATH")
+
+        self.runner = runner
+        self.host = host
+        # port is unused, since this is local
+        self.port = port
+
+    def connect(self, port=None):
+        ''' connect to the chroot; nothing to do here '''
+
+        vvv("THIS IS A LOCAL CHROOT DIR", host=self.chroot)
+
+        return self
+
+    def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, executable='/bin/sh', in_data=None, su=None, su_user=None):
+        ''' run a command on the chroot '''
+
+        if su or su_user:
+            raise errors.AnsibleError("Internal Error: this module does not support running commands via su")
+
+        if in_data:
+            raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
+
+        # We enter chroot as root so sudo stuff can be ignored
+
+        if executable:
+            local_cmd = [self.chroot_cmd, self.chroot, executable, '-c', cmd]
+        else:
+            local_cmd = '%s "%s" %s' % (self.chroot_cmd, self.chroot, cmd)
+
+        vvv("EXEC %s" % (local_cmd), host=self.chroot)
+        p = subprocess.Popen(local_cmd, shell=isinstance(local_cmd, basestring),
+                             cwd=self.runner.basedir,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        stdout, stderr = p.communicate()
+        return (p.returncode, '', stdout, stderr)
+
+    def put_file(self, in_path, out_path):
+        ''' transfer a file from local to chroot '''
+
+        if not out_path.startswith(os.path.sep):
+            out_path = os.path.join(os.path.sep, out_path)
+        normpath = os.path.normpath(out_path)
+        out_path = os.path.join(self.chroot, normpath[1:])
+
+        vvv("PUT %s TO %s" % (in_path, out_path), host=self.chroot)
+        if not os.path.exists(in_path):
+            raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+        try:
+            shutil.copyfile(in_path, out_path)
+        except shutil.Error:
+            traceback.print_exc()
+            raise errors.AnsibleError("failed to copy: %s and %s are the same" % (in_path, out_path))
+        except IOError:
+            traceback.print_exc()
+            raise errors.AnsibleError("failed to transfer file to %s" % out_path)
+
+    def fetch_file(self, in_path, out_path):
+        ''' fetch a file from chroot to local '''
+
+        if not in_path.startswith(os.path.sep):
+            in_path = os.path.join(os.path.sep, in_path)
+        normpath = os.path.normpath(in_path)
+        in_path = os.path.join(self.chroot, normpath[1:])
+
+        vvv("FETCH %s TO %s" % (in_path, out_path), host=self.chroot)
+        if not os.path.exists(in_path):
+            raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+        try:
+            shutil.copyfile(in_path, out_path)
+        except shutil.Error:
+            traceback.print_exc()
+            raise errors.AnsibleError("failed to copy: %s and %s are the same" % (in_path, out_path))
+        except IOError:
+            traceback.print_exc()
+            raise errors.AnsibleError("failed to transfer file to %s" % out_path)
+
+    def close(self):
+        ''' terminate the connection; nothing to do here '''
+        pass

v2/ansible/plugins/connections/fireball.py

@@ -0,0 +1,151 @@
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import os
+import base64
+from ansible.callbacks import vvv
+from ansible import utils
+from ansible import errors
+from ansible import constants
+
+HAVE_ZMQ=False
+
+try:
+    import zmq
+    HAVE_ZMQ=True
+except ImportError:
+    pass
+
+class Connection(object):
+    ''' ZeroMQ accelerated connection '''
+
+    def __init__(self, runner, host, port, *args, **kwargs):
+
+        self.runner = runner
+        self.has_pipelining = False
+
+        # attempt to work around shared-memory funness
+        if getattr(self.runner, 'aes_keys', None):
+            utils.AES_KEYS = self.runner.aes_keys
+
+        self.host = host
+        self.key = utils.key_for_hostname(host)
+        self.context = None
+        self.socket = None
+
+        if  port is None:
+            self.port = constants.ZEROMQ_PORT
+        else:
+            self.port = port
+
+    def connect(self):
+        ''' activates the connection object '''
+
+        if not HAVE_ZMQ:
+            raise errors.AnsibleError("zmq is not installed")
+        
+        # this is rough/temporary and will likely be optimized later ...
+        self.context = zmq.Context()
+        socket = self.context.socket(zmq.REQ)
+        addr = "tcp://%s:%s" % (self.host, self.port)
+        socket.connect(addr)
+        self.socket = socket    
+
+        return self
+
+    def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None, su_user=None, su=None):
+        ''' run a command on the remote host '''
+
+        if in_data:
+            raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
+
+        vvv("EXEC COMMAND %s" % cmd)
+
+        if (self.runner.sudo and sudoable) or (self.runner.su and su):
+            raise errors.AnsibleError(
+                "When using fireball, do not specify sudo or su to run your tasks. " +
+                "Instead sudo the fireball action with sudo. " +
+                "Task will communicate with the fireball already running in sudo mode."
+            )
+
+        data = dict(
+            mode='command',
+            cmd=cmd,
+            tmp_path=tmp_path,
+            executable=executable,
+        )
+        data = utils.jsonify(data)
+        data = utils.encrypt(self.key, data)
+        self.socket.send(data)
+        
+        response = self.socket.recv()
+        response = utils.decrypt(self.key, response)
+        response = utils.parse_json(response)
+
+        return (response.get('rc',None), '', response.get('stdout',''), response.get('stderr',''))
+
+    def put_file(self, in_path, out_path):
+
+        ''' transfer a file from local to remote '''
+        vvv("PUT %s TO %s" % (in_path, out_path), host=self.host)
+
+        if not os.path.exists(in_path):
+            raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+        data = file(in_path).read()
+        data = base64.b64encode(data)
+
+        data = dict(mode='put', data=data, out_path=out_path)
+        # TODO: support chunked file transfer
+        data = utils.jsonify(data)
+        data = utils.encrypt(self.key, data)
+        self.socket.send(data)
+
+        response = self.socket.recv()
+        response = utils.decrypt(self.key, response)
+        response = utils.parse_json(response)
+
+        # no meaningful response needed for this
+
+    def fetch_file(self, in_path, out_path):
+        ''' save a remote file to the specified path '''
+        vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host)
+
+        data = dict(mode='fetch', in_path=in_path)
+        data = utils.jsonify(data)
+        data = utils.encrypt(self.key, data)
+        self.socket.send(data)
+
+        response = self.socket.recv()
+        response = utils.decrypt(self.key, response)
+        response = utils.parse_json(response)
+        response = response['data']
+        response = base64.b64decode(response)        
+
+        fh = open(out_path, "w")
+        fh.write(response)
+        fh.close()
+
+    def close(self):
+        ''' terminate the connection '''
+        # Be a good citizen
+        try:
+            self.socket.close()
+            self.context.term()
+        except:
+            pass
+

v2/ansible/plugins/connections/funcd.py

@@ -0,0 +1,99 @@
+# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+# Based on chroot.py (c) 2013, Maykel Moya <mmoya@speedyrails.com>
+# (c) 2013, Michael Scherer <misc@zarb.org>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# ---
+# The func transport permit to use ansible over func. For people who have already setup
+# func and that wish to play with ansible, this permit to move gradually to ansible
+# without having to redo completely the setup of the network.
+
+HAVE_FUNC=False
+try:
+    import func.overlord.client as fc
+    HAVE_FUNC=True
+except ImportError:
+    pass
+
+import os
+from ansible.callbacks import vvv
+from ansible import errors
+import tempfile
+import shutil
+
+
+class Connection(object):
+    ''' Func-based connections '''
+
+    def __init__(self, runner, host, port, *args, **kwargs):
+        self.runner = runner
+        self.host = host
+        self.has_pipelining = False
+        # port is unused, this go on func
+        self.port = port
+
+    def connect(self, port=None):
+        if not HAVE_FUNC:
+            raise errors.AnsibleError("func is not installed")
+
+        self.client = fc.Client(self.host)
+        return self
+
+    def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False,
+                     executable='/bin/sh', in_data=None, su=None, su_user=None):
+        ''' run a command on the remote minion '''
+
+        if su or su_user:
+            raise errors.AnsibleError("Internal Error: this module does not support running commands via su")
+
+        if in_data:
+            raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
+
+        vvv("EXEC %s" % (cmd), host=self.host)
+        p = self.client.command.run(cmd)[self.host]
+        return (p[0], '', p[1], p[2])
+
+    def _normalize_path(self, path, prefix):
+        if not path.startswith(os.path.sep):
+            path = os.path.join(os.path.sep, path)
+        normpath = os.path.normpath(path)
+        return os.path.join(prefix, normpath[1:])
+
+    def put_file(self, in_path, out_path):
+        ''' transfer a file from local to remote '''
+
+        out_path = self._normalize_path(out_path, '/')
+        vvv("PUT %s TO %s" % (in_path, out_path), host=self.host)
+        self.client.local.copyfile.send(in_path, out_path)
+
+    def fetch_file(self, in_path, out_path):
+        ''' fetch a file from remote to local '''
+
+        in_path = self._normalize_path(in_path, '/')
+        vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host)
+        # need to use a tmp dir due to difference of semantic for getfile
+        # ( who take a # directory as destination) and fetch_file, who
+        # take a file directly
+        tmpdir = tempfile.mkdtemp(prefix="func_ansible")
+        self.client.local.getfile.get(in_path, tmpdir)
+        shutil.move(os.path.join(tmpdir, self.host, os.path.basename(in_path)),
+                    out_path)
+        shutil.rmtree(tmpdir)
+
+    def close(self):
+        ''' terminate the connection; nothing to do here '''
+        pass

v2/ansible/plugins/connections/jail.py

@@ -0,0 +1,151 @@
+# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+# and chroot.py     (c) 2013, Maykel Moya <mmoya@speedyrails.com>
+# (c) 2013, Michael Scherer <misc@zarb.org>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import distutils.spawn
+import traceback
+import os
+import shutil
+import subprocess
+from ansible import errors
+from ansible.callbacks import vvv
+
+class Connection(object):
+    ''' Local chroot based connections '''
+
+    def _search_executable(self, executable):
+        cmd = distutils.spawn.find_executable(executable)
+        if not cmd:
+            raise errors.AnsibleError("%s command not found in PATH") % executable
+        return cmd
+
+    def list_jails(self):
+        p = subprocess.Popen([self.jls_cmd, '-q', 'name'],
+                             cwd=self.runner.basedir,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        stdout, stderr = p.communicate()
+
+        return stdout.split()
+
+    def get_jail_path(self):
+        p = subprocess.Popen([self.jls_cmd, '-j', self.jail, '-q', 'path'],
+                             cwd=self.runner.basedir,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        stdout, stderr = p.communicate()
+        # remove \n
+        return stdout[:-1]
+
+ 
+        
+    def __init__(self, runner, host, port, *args, **kwargs):
+        self.jail = host
+        self.runner = runner
+        self.host = host
+        self.has_pipelining = False
+
+        if os.geteuid() != 0:
+            raise errors.AnsibleError("jail connection requires running as root")
+
+        self.jls_cmd = self._search_executable('jls')
+        self.jexec_cmd = self._search_executable('jexec')
+        
+        if not self.jail in self.list_jails():
+            raise errors.AnsibleError("incorrect jail name %s" % self.jail)
+
+
+        self.host = host
+        # port is unused, since this is local
+        self.port = port
+
+    def connect(self, port=None):
+        ''' connect to the chroot; nothing to do here '''
+
+        vvv("THIS IS A LOCAL CHROOT DIR", host=self.jail)
+
+        return self
+
+    # a modifier
+    def _generate_cmd(self, executable, cmd):
+        if executable:
+            local_cmd = [self.jexec_cmd, self.jail, executable, '-c', cmd]
+        else:
+            local_cmd = '%s "%s" %s' % (self.jexec_cmd, self.jail, cmd)
+        return local_cmd
+
+    def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, executable='/bin/sh', in_data=None, su=None, su_user=None):
+        ''' run a command on the chroot '''
+
+        if su or su_user:
+            raise errors.AnsibleError("Internal Error: this module does not support running commands via su")
+
+        if in_data:
+            raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
+
+        # We enter chroot as root so sudo stuff can be ignored
+        local_cmd = self._generate_cmd(executable, cmd)
+
+        vvv("EXEC %s" % (local_cmd), host=self.jail)
+        p = subprocess.Popen(local_cmd, shell=isinstance(local_cmd, basestring),
+                             cwd=self.runner.basedir,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        stdout, stderr = p.communicate()
+        return (p.returncode, '', stdout, stderr)
+
+    def _normalize_path(self, path, prefix):
+        if not path.startswith(os.path.sep):
+            path = os.path.join(os.path.sep, path)
+        normpath = os.path.normpath(path)
+        return os.path.join(prefix, normpath[1:])
+
+    def _copy_file(self, in_path, out_path):
+        if not os.path.exists(in_path):
+            raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+        try:
+            shutil.copyfile(in_path, out_path)
+        except shutil.Error:
+            traceback.print_exc()
+            raise errors.AnsibleError("failed to copy: %s and %s are the same" % (in_path, out_path))
+        except IOError:
+            traceback.print_exc()
+            raise errors.AnsibleError("failed to transfer file to %s" % out_path)
+
+    def put_file(self, in_path, out_path):
+        ''' transfer a file from local to chroot '''
+
+        out_path = self._normalize_path(out_path, self.get_jail_path())
+        vvv("PUT %s TO %s" % (in_path, out_path), host=self.jail)
+
+        self._copy_file(in_path, out_path)
+
+    def fetch_file(self, in_path, out_path):
+        ''' fetch a file from chroot to local '''
+
+        in_path = self._normalize_path(in_path, self.get_jail_path())
+        vvv("FETCH %s TO %s" % (in_path, out_path), host=self.jail)
+
+        self._copy_file(in_path, out_path)
+
+    def close(self):
+        ''' terminate the connection; nothing to do here '''
+        pass

v2/ansible/plugins/connections/libvirt_lxc.py

@@ -0,0 +1,127 @@
+# Based on local.py (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+# Based on chroot.py (c) 2013, Maykel Moya <mmoya@speedyrails.com>
+# (c) 2013, Michael Scherer <misc@zarb.org>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import distutils.spawn
+import os
+import subprocess
+from ansible import errors
+from ansible.callbacks import vvv
+
+class Connection(object):
+    ''' Local lxc based connections '''
+
+    def _search_executable(self, executable):
+        cmd = distutils.spawn.find_executable(executable)
+        if not cmd:
+            raise errors.AnsibleError("%s command not found in PATH") % executable
+        return cmd
+
+    def _check_domain(self, domain):
+        p = subprocess.Popen([self.cmd, '-q', '-c', 'lxc:///', 'dominfo', domain],
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        p.communicate()
+        if p.returncode:
+            raise errors.AnsibleError("%s is not a lxc defined in libvirt" % domain)
+
+    def __init__(self, runner, host, port, *args, **kwargs):
+        self.lxc = host
+
+        self.cmd = self._search_executable('virsh')
+
+        self._check_domain(host)
+
+        self.runner = runner
+        self.host = host
+        # port is unused, since this is local
+        self.port = port
+
+    def connect(self, port=None):
+        ''' connect to the lxc; nothing to do here '''
+
+        vvv("THIS IS A LOCAL LXC DIR", host=self.lxc)
+
+        return self
+
+    def _generate_cmd(self, executable, cmd):
+        if executable:
+            local_cmd = [self.cmd, '-q', '-c', 'lxc:///', 'lxc-enter-namespace', self.lxc, '--', executable , '-c', cmd]
+        else:
+            local_cmd = '%s -q -c lxc:/// lxc-enter-namespace %s -- %s' % (self.cmd, self.lxc, cmd)
+        return local_cmd
+
+    def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None, su=None, su_user=None):
+        ''' run a command on the chroot '''
+
+        if su or su_user:
+            raise errors.AnsibleError("Internal Error: this module does not support running commands via su")
+
+        if in_data:
+            raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
+
+        # We enter lxc as root so sudo stuff can be ignored
+        local_cmd = self._generate_cmd(executable, cmd)
+
+        vvv("EXEC %s" % (local_cmd), host=self.lxc)
+        p = subprocess.Popen(local_cmd, shell=isinstance(local_cmd, basestring),
+                             cwd=self.runner.basedir,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        stdout, stderr = p.communicate()
+        return (p.returncode, '', stdout, stderr)
+
+    def _normalize_path(self, path, prefix):
+        if not path.startswith(os.path.sep):
+            path = os.path.join(os.path.sep, path)
+        normpath = os.path.normpath(path)
+        return os.path.join(prefix, normpath[1:])
+
+    def put_file(self, in_path, out_path):
+        ''' transfer a file from local to lxc '''
+
+        out_path = self._normalize_path(out_path, '/')
+        vvv("PUT %s TO %s" % (in_path, out_path), host=self.lxc)
+        
+        local_cmd = [self.cmd, '-q', '-c', 'lxc:///', 'lxc-enter-namespace', self.lxc, '--', '/bin/tee', out_path]
+        vvv("EXEC %s" % (local_cmd), host=self.lxc)
+
+        p = subprocess.Popen(local_cmd, cwd=self.runner.basedir,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE) 
+        stdout, stderr = p.communicate(open(in_path,'rb').read())
+ 
+    def fetch_file(self, in_path, out_path):
+        ''' fetch a file from lxc to local '''
+
+        in_path = self._normalize_path(in_path, '/')
+        vvv("FETCH %s TO %s" % (in_path, out_path), host=self.lxc)
+
+        local_cmd = [self.cmd, '-q', '-c', 'lxc:///', 'lxc-enter-namespace', self.lxc, '--', '/bin/cat', in_path]
+        vvv("EXEC %s" % (local_cmd), host=self.lxc)
+
+        p = subprocess.Popen(local_cmd, cwd=self.runner.basedir,
+                             stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        open(out_path,'wb').write(stdout)
+
+
+    def close(self):
+        ''' terminate the connection; nothing to do here '''
+        pass

v2/ansible/plugins/connections/local.py

@@ -0,0 +1,138 @@
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import traceback
+import os
+import pipes
+import shutil
+import subprocess
+import select
+import fcntl
+
+from ansible.errors import AnsibleError
+from ansible.plugins.connections import ConnectionBase
+
+from ansible.utils.debug import debug
+
+class Connection(ConnectionBase):
+    ''' Local based connections '''
+
+    def get_transport(self):
+        ''' used to identify this connection object '''
+        return 'local'
+
+    def connect(self, port=None):
+        ''' connect to the local host; nothing to do here '''
+        return self
+
+    def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, executable='/bin/sh', in_data=None, su=None, su_user=None):
+        ''' run a command on the local host '''
+
+        debug("in local.exec_command()")
+        # su requires to be run from a terminal, and therefore isn't supported here (yet?)
+        if su or su_user:
+            raise AnsibleError("Internal Error: this module does not support running commands via su")
+
+        if in_data:
+            raise AnsibleError("Internal Error: this module does not support optimized module pipelining")
+
+        # FIXME: su/sudo stuff needs to be generalized
+        #if not self.runner.sudo or not sudoable:
+        #    if executable:
+        #        local_cmd = executable.split() + ['-c', cmd]
+        #    else:
+        #        local_cmd = cmd
+        #else:
+        #    local_cmd, prompt, success_key = utils.make_sudo_cmd(self.runner.sudo_exe, sudo_user, executable, cmd)
+        if executable:
+            local_cmd = executable.split() + ['-c', cmd]
+        else:
+            local_cmd = cmd
+
+        executable = executable.split()[0] if executable else None
+
+        self._display.vvv("%s EXEC %s" % (self._host, local_cmd))
+        # FIXME: cwd= needs to be set to the basedir of the playbook
+        debug("opening command with Popen()")
+        p = subprocess.Popen(
+            local_cmd,
+            shell=isinstance(local_cmd, basestring),
+            executable=executable, #cwd=...
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        debug("done running command with Popen()")
+
+        # FIXME: more su/sudo stuff
+        #if self.runner.sudo and sudoable and self.runner.sudo_pass:
+        #    fcntl.fcntl(p.stdout, fcntl.F_SETFL,
+        #                fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
+        #    fcntl.fcntl(p.stderr, fcntl.F_SETFL,
+        #                fcntl.fcntl(p.stderr, fcntl.F_GETFL) | os.O_NONBLOCK)
+        #    sudo_output = ''
+        #    while not sudo_output.endswith(prompt) and success_key not in sudo_output:
+        #        rfd, wfd, efd = select.select([p.stdout, p.stderr], [],
+        #                                      [p.stdout, p.stderr], self.runner.timeout)
+        #        if p.stdout in rfd:
+        #            chunk = p.stdout.read()
+        #        elif p.stderr in rfd:
+        #            chunk = p.stderr.read()
+        #        else:
+        #            stdout, stderr = p.communicate()
+        #            raise AnsibleError('timeout waiting for sudo password prompt:\n' + sudo_output)
+        #        if not chunk:
+        #            stdout, stderr = p.communicate()
+        #            raise AnsibleError('sudo output closed while waiting for password prompt:\n' + sudo_output)
+        #        sudo_output += chunk
+        #    if success_key not in sudo_output:
+        #        p.stdin.write(self.runner.sudo_pass + '\n')
+        #    fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK)
+        #    fcntl.fcntl(p.stderr, fcntl.F_SETFL, fcntl.fcntl(p.stderr, fcntl.F_GETFL) & ~os.O_NONBLOCK)
+
+        debug("getting output with communicate()")
+        stdout, stderr = p.communicate()
+        debug("done communicating")
+
+        debug("done with local.exec_command()")
+        return (p.returncode, '', stdout, stderr)
+
+    def put_file(self, in_path, out_path):
+        ''' transfer a file from local to local '''
+
+        #vvv("PUT %s TO %s" % (in_path, out_path), host=self.host)
+        self._display.vvv("%s PUT %s TO %s" % (self._host, in_path, out_path))
+        if not os.path.exists(in_path):
+            raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+        try:
+            shutil.copyfile(in_path, out_path)
+        except shutil.Error:
+            traceback.print_exc()
+            raise AnsibleError("failed to copy: %s and %s are the same" % (in_path, out_path))
+        except IOError:
+            traceback.print_exc()
+            raise AnsibleError("failed to transfer file to %s" % out_path)
+
+    def fetch_file(self, in_path, out_path):
+        #vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host)
+        self._display.vvv("%s FETCH %s TO %s" % (self._host, in_path, out_path))
+        ''' fetch a file from local to local -- for copatibility '''
+        self.put_file(in_path, out_path)
+
+    def close(self):
+        ''' terminate the connection; nothing to do here '''
+        pass

v2/ansible/plugins/connections/paramiko_ssh.py

@@ -0,0 +1,417 @@
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# ---
+# The paramiko transport is provided because many distributions, in particular EL6 and before
+# do not support ControlPersist in their SSH implementations.  This is needed on the Ansible
+# control machine to be reasonably efficient with connections.  Thus paramiko is faster
+# for most users on these platforms.  Users with ControlPersist capability can consider
+# using -c ssh or configuring the transport in ansible.cfg.
+
+import warnings
+import os
+import pipes
+import socket
+import random
+import logging
+import tempfile
+import traceback
+import fcntl
+import re
+import sys
+from termios import tcflush, TCIFLUSH
+from binascii import hexlify
+from ansible.callbacks import vvv
+from ansible import errors
+from ansible import utils
+from ansible import constants as C
+
+AUTHENTICITY_MSG="""
+paramiko: The authenticity of host '%s' can't be established.
+The %s key fingerprint is %s.
+Are you sure you want to continue connecting (yes/no)?
+"""
+
+# prevent paramiko warning noise -- see http://stackoverflow.com/questions/3920502/
+HAVE_PARAMIKO=False
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore")
+    try:
+        import paramiko
+        HAVE_PARAMIKO=True
+        logging.getLogger("paramiko").setLevel(logging.WARNING)
+    except ImportError:
+        pass
+
+class MyAddPolicy(object):
+    """
+    Based on AutoAddPolicy in paramiko so we can determine when keys are added
+    and also prompt for input.
+
+    Policy for automatically adding the hostname and new host key to the
+    local L{HostKeys} object, and saving it.  This is used by L{SSHClient}.
+    """
+
+    def __init__(self, runner):
+        self.runner = runner
+
+    def missing_host_key(self, client, hostname, key):
+
+        if C.HOST_KEY_CHECKING:
+
+            fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX)
+            fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_EX)
+
+            old_stdin = sys.stdin
+            sys.stdin = self.runner._new_stdin
+            fingerprint = hexlify(key.get_fingerprint())
+            ktype = key.get_name()
+
+            # clear out any premature input on sys.stdin
+            tcflush(sys.stdin, TCIFLUSH)
+
+            inp = raw_input(AUTHENTICITY_MSG % (hostname, ktype, fingerprint))
+            sys.stdin = old_stdin
+            if inp not in ['yes','y','']:
+                fcntl.flock(self.runner.output_lockfile, fcntl.LOCK_UN)
+                fcntl.flock(self.runner.process_lockfile, fcntl.LOCK_UN)
+                raise errors.AnsibleError("host connection rejected by user")
+
+            fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_UN)
+            fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_UN)
+
+
+        key._added_by_ansible_this_time = True
+
+        # existing implementation below:
+        client._host_keys.add(hostname, key.get_name(), key)
+
+        # host keys are actually saved in close() function below
+        # in order to control ordering.
+
+
+# keep connection objects on a per host basis to avoid repeated attempts to reconnect
+
+SSH_CONNECTION_CACHE = {}
+SFTP_CONNECTION_CACHE = {}
+
+class Connection(object):
+    ''' SSH based connections with Paramiko '''
+
+    def __init__(self, runner, host, port, user, password, private_key_file, *args, **kwargs):
+
+        self.ssh = None
+        self.sftp = None
+        self.runner = runner
+        self.host = host
+        self.port = port or 22
+        self.user = user
+        self.password = password
+        self.private_key_file = private_key_file
+        self.has_pipelining = False
+
+    def _cache_key(self):
+        return "%s__%s__" % (self.host, self.user)
+
+    def connect(self):
+        cache_key = self._cache_key()
+        if cache_key in SSH_CONNECTION_CACHE:
+            self.ssh = SSH_CONNECTION_CACHE[cache_key]
+        else:
+            self.ssh = SSH_CONNECTION_CACHE[cache_key] = self._connect_uncached()
+        return self
+
+    def _connect_uncached(self):
+        ''' activates the connection object '''
+
+        if not HAVE_PARAMIKO:
+            raise errors.AnsibleError("paramiko is not installed")
+
+        vvv("ESTABLISH CONNECTION FOR USER: %s on PORT %s TO %s" % (self.user, self.port, self.host), host=self.host)
+
+        ssh = paramiko.SSHClient()
+
+        self.keyfile = os.path.expanduser("~/.ssh/known_hosts")
+
+        if C.HOST_KEY_CHECKING:
+            ssh.load_system_host_keys()
+
+        ssh.set_missing_host_key_policy(MyAddPolicy(self.runner))
+
+        allow_agent = True
+
+        if self.password is not None:
+            allow_agent = False
+
+        try:
+
+            if self.private_key_file:
+                key_filename = os.path.expanduser(self.private_key_file)
+            elif self.runner.private_key_file:
+                key_filename = os.path.expanduser(self.runner.private_key_file)
+            else:
+                key_filename = None
+            ssh.connect(self.host, username=self.user, allow_agent=allow_agent, look_for_keys=True,
+                key_filename=key_filename, password=self.password,
+                timeout=self.runner.timeout, port=self.port)
+
+        except Exception, e:
+
+            msg = str(e)
+            if "PID check failed" in msg:
+                raise errors.AnsibleError("paramiko version issue, please upgrade paramiko on the machine running ansible")
+            elif "Private key file is encrypted" in msg:
+                msg = 'ssh %s@%s:%s : %s\nTo connect as a different user, use -u <username>.' % (
+                    self.user, self.host, self.port, msg)
+                raise errors.AnsibleConnectionFailed(msg)
+            else:
+                raise errors.AnsibleConnectionFailed(msg)
+
+        return ssh
+
+    def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, executable='/bin/sh', in_data=None, su=None, su_user=None):
+        ''' run a command on the remote host '''
+
+        if in_data:
+            raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
+
+        bufsize = 4096
+
+        try:
+
+            self.ssh.get_transport().set_keepalive(5)
+            chan = self.ssh.get_transport().open_session()
+
+        except Exception, e:
+
+            msg = "Failed to open session"
+            if len(str(e)) > 0:
+                msg += ": %s" % str(e)
+            raise errors.AnsibleConnectionFailed(msg)
+
+        no_prompt_out = ''
+        no_prompt_err = ''
+        if not (self.runner.sudo and sudoable) and not (self.runner.su and su):
+
+            if executable:
+                quoted_command = executable + ' -c ' + pipes.quote(cmd)
+            else:
+                quoted_command = cmd
+            vvv("EXEC %s" % quoted_command, host=self.host)
+            chan.exec_command(quoted_command)
+
+        else:
+
+            # sudo usually requires a PTY (cf. requiretty option), therefore
+            # we give it one by default (pty=True in ansble.cfg), and we try
+            # to initialise from the calling environment
+            if C.PARAMIKO_PTY:
+                chan.get_pty(term=os.getenv('TERM', 'vt100'),
+                             width=int(os.getenv('COLUMNS', 0)),
+                             height=int(os.getenv('LINES', 0)))
+            if self.runner.sudo or sudoable:
+                shcmd, prompt, success_key = utils.make_sudo_cmd(self.runner.sudo_exe, sudo_user, executable, cmd)
+            elif self.runner.su or su:
+                shcmd, prompt, success_key = utils.make_su_cmd(su_user, executable, cmd)
+
+            vvv("EXEC %s" % shcmd, host=self.host)
+            sudo_output = ''
+
+            try:
+
+                chan.exec_command(shcmd)
+
+                if self.runner.sudo_pass or self.runner.su_pass:
+
+                    while True:
+
+                        if success_key in sudo_output or \
+                            (self.runner.sudo_pass and sudo_output.endswith(prompt)) or \
+                            (self.runner.su_pass and utils.su_prompts.check_su_prompt(sudo_output)):
+                            break
+                        chunk = chan.recv(bufsize)
+
+                        if not chunk:
+                            if 'unknown user' in sudo_output:
+                                raise errors.AnsibleError(
+                                    'user %s does not exist' % sudo_user)
+                            else:
+                                raise errors.AnsibleError('ssh connection ' +
+                                    'closed waiting for password prompt')
+                        sudo_output += chunk
+
+                    if success_key not in sudo_output:
+
+                        if sudoable:
+                            chan.sendall(self.runner.sudo_pass + '\n')
+                        elif su:
+                            chan.sendall(self.runner.su_pass + '\n')
+                    else:
+                        no_prompt_out += sudo_output
+                        no_prompt_err += sudo_output
+
+            except socket.timeout:
+
+                raise errors.AnsibleError('ssh timed out waiting for sudo.\n' + sudo_output)
+
+        stdout = ''.join(chan.makefile('rb', bufsize))
+        stderr = ''.join(chan.makefile_stderr('rb', bufsize))
+
+        return (chan.recv_exit_status(), '', no_prompt_out + stdout, no_prompt_out + stderr)
+
+    def put_file(self, in_path, out_path):
+        ''' transfer a file from local to remote '''
+
+        vvv("PUT %s TO %s" % (in_path, out_path), host=self.host)
+
+        if not os.path.exists(in_path):
+            raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+
+        try:
+            self.sftp = self.ssh.open_sftp()
+        except Exception, e:
+            raise errors.AnsibleError("failed to open a SFTP connection (%s)" % e)
+
+        try:
+            self.sftp.put(in_path, out_path)
+        except IOError:
+            raise errors.AnsibleError("failed to transfer file to %s" % out_path)
+
+    def _connect_sftp(self):
+
+        cache_key = "%s__%s__" % (self.host, self.user)
+        if cache_key in SFTP_CONNECTION_CACHE:
+            return SFTP_CONNECTION_CACHE[cache_key]
+        else:
+            result = SFTP_CONNECTION_CACHE[cache_key] = self.connect().ssh.open_sftp()
+            return result
+
+    def fetch_file(self, in_path, out_path):
+        ''' save a remote file to the specified path '''
+
+        vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host)
+
+        try:
+            self.sftp = self._connect_sftp()
+        except Exception, e:
+            raise errors.AnsibleError("failed to open a SFTP connection (%s)", e)
+
+        try:
+            self.sftp.get(in_path, out_path)
+        except IOError:
+            raise errors.AnsibleError("failed to transfer file from %s" % in_path)
+
+    def _any_keys_added(self):
+
+        added_any = False
+        for hostname, keys in self.ssh._host_keys.iteritems():
+            for keytype, key in keys.iteritems():
+                added_this_time = getattr(key, '_added_by_ansible_this_time', False)
+                if added_this_time:
+                    return True
+        return False
+
+    def _save_ssh_host_keys(self, filename):
+        '''
+        not using the paramiko save_ssh_host_keys function as we want to add new SSH keys at the bottom so folks
+        don't complain about it :)
+        '''
+
+        if not self._any_keys_added():
+            return False
+
+        path = os.path.expanduser("~/.ssh")
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+        f = open(filename, 'w')
+
+        for hostname, keys in self.ssh._host_keys.iteritems():
+
+            for keytype, key in keys.iteritems():
+
+                # was f.write
+                added_this_time = getattr(key, '_added_by_ansible_this_time', False)
+                if not added_this_time:
+                    f.write("%s %s %s\n" % (hostname, keytype, key.get_base64()))
+
+        for hostname, keys in self.ssh._host_keys.iteritems():
+
+            for keytype, key in keys.iteritems():
+                added_this_time = getattr(key, '_added_by_ansible_this_time', False)
+                if added_this_time:
+                    f.write("%s %s %s\n" % (hostname, keytype, key.get_base64()))
+
+        f.close()
+
+    def close(self):
+        ''' terminate the connection '''
+
+        cache_key = self._cache_key()
+        SSH_CONNECTION_CACHE.pop(cache_key, None)
+        SFTP_CONNECTION_CACHE.pop(cache_key, None)
+
+        if self.sftp is not None:
+            self.sftp.close()
+
+        if C.HOST_KEY_CHECKING and C.PARAMIKO_RECORD_HOST_KEYS and self._any_keys_added():
+
+            # add any new SSH host keys -- warning -- this could be slow
+            lockfile = self.keyfile.replace("known_hosts",".known_hosts.lock")
+            dirname = os.path.dirname(self.keyfile)
+            if not os.path.exists(dirname):
+                os.makedirs(dirname)
+
+            KEY_LOCK = open(lockfile, 'w')
+            fcntl.lockf(KEY_LOCK, fcntl.LOCK_EX)
+
+            try:
+                # just in case any were added recently
+
+                self.ssh.load_system_host_keys()
+                self.ssh._host_keys.update(self.ssh._system_host_keys)
+
+                # gather information about the current key file, so
+                # we can ensure the new file has the correct mode/owner
+
+                key_dir  = os.path.dirname(self.keyfile)
+                key_stat = os.stat(self.keyfile)
+
+                # Save the new keys to a temporary file and move it into place
+                # rather than rewriting the file. We set delete=False because
+                # the file will be moved into place rather than cleaned up.
+
+                tmp_keyfile = tempfile.NamedTemporaryFile(dir=key_dir, delete=False)
+                os.chmod(tmp_keyfile.name, key_stat.st_mode & 07777)
+                os.chown(tmp_keyfile.name, key_stat.st_uid, key_stat.st_gid)
+
+                self._save_ssh_host_keys(tmp_keyfile.name)
+                tmp_keyfile.close()
+
+                os.rename(tmp_keyfile.name, self.keyfile)
+
+            except:
+
+                # unable to save keys, including scenario when key was invalid
+                # and caught earlier
+                traceback.print_exc()
+                pass
+            fcntl.lockf(KEY_LOCK, fcntl.LOCK_UN)
+
+        self.ssh.close()
+

v2/ansible/plugins/connections/ssh.py

@@ -0,0 +1,487 @@
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import re
+import subprocess
+import shlex
+import pipes
+import random
+import select
+import fcntl
+import hmac
+import pwd
+import gettext
+import pty
+from hashlib import sha1
+
+from ansible import constants as C
+from ansible.errors import AnsibleError, AnsibleConnectionFailure
+from ansible.plugins.connections import ConnectionBase
+
+class Connection(ConnectionBase):
+    ''' ssh based connections '''
+
+    def __init__(self, host, connection_info, *args, **kwargs):
+        super(Connection, self).__init__(host, connection_info)
+
+        # SSH connection specific init stuff
+        self.HASHED_KEY_MAGIC = "|1|"
+        self._has_pipelining = True
+
+        # FIXME: move the lockfile locations to ActionBase?
+        #fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX)
+        #self.cp_dir = utils.prepare_writeable_dir('$HOME/.ansible/cp',mode=0700)
+        self._cp_dir = '/tmp'
+        #fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_UN)
+
+    def get_transport(self):
+        ''' used to identify this connection object from other classes '''
+        return 'ssh'
+
+    def connect(self):
+        ''' connect to the remote host '''
+
+        self._display.vvv("ESTABLISH CONNECTION FOR USER: %s" % self._connection_info.remote_user, host=self._host)
+
+        self._common_args = []
+        extra_args = C.ANSIBLE_SSH_ARGS
+        if extra_args is not None:
+            # make sure there is no empty string added as this can produce weird errors
+            self._common_args += [x.strip() for x in shlex.split(extra_args) if x.strip()]
+        else:
+            self._common_args += [
+                "-o", "ControlMaster=auto",
+                "-o", "ControlPersist=60s",
+                "-o", "ControlPath=\"%s\"" % (C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=self._cp_dir)),
+            ]
+
+        cp_in_use = False
+        cp_path_set = False
+        for arg in self._common_args:
+            if "ControlPersist" in arg:
+                cp_in_use = True
+            if "ControlPath" in arg:
+                cp_path_set = True
+
+        if cp_in_use and not cp_path_set:
+            self._common_args += ["-o", "ControlPath=\"%s\"" % (C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=self._cp_dir))]
+
+        if not C.HOST_KEY_CHECKING:
+            self._common_args += ["-o", "StrictHostKeyChecking=no"]
+
+        if self._connection_info.port is not None:
+            self._common_args += ["-o", "Port=%d" % (self._connection_info.port)]
+        #if self.private_key_file is not None:
+        #    self._common_args += ["-o", "IdentityFile=\"%s\"" % os.path.expanduser(self.private_key_file)]
+        #elif self.runner.private_key_file is not None:
+        #    self._common_args += ["-o", "IdentityFile=\"%s\"" % os.path.expanduser(self.runner.private_key_file)]
+        if self._connection_info.password:
+            self._common_args += ["-o", "GSSAPIAuthentication=no",
+                                 "-o", "PubkeyAuthentication=no"]
+        else:
+            self._common_args += ["-o", "KbdInteractiveAuthentication=no",
+                                 "-o", "PreferredAuthentications=gssapi-with-mic,gssapi-keyex,hostbased,publickey",
+                                 "-o", "PasswordAuthentication=no"]
+        if self._connection_info.remote_user != pwd.getpwuid(os.geteuid())[0]:
+            self._common_args += ["-o", "User="+self._connection_info.remote_user]
+        # FIXME: figure out where this goes
+        #self._common_args += ["-o", "ConnectTimeout=%d" % self.runner.timeout]
+        self._common_args += ["-o", "ConnectTimeout=15"]
+
+        return self
+
+    def _run(self, cmd, indata):
+        if indata:
+            # do not use pseudo-pty
+            p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            stdin = p.stdin
+        else:
+            # try to use upseudo-pty
+            try:
+                # Make sure stdin is a proper (pseudo) pty to avoid: tcgetattr errors
+                master, slave = pty.openpty()
+                p = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                stdin = os.fdopen(master, 'w', 0)
+                os.close(slave)
+            except:
+                p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                stdin = p.stdin
+
+        return (p, stdin)
+
+    def _password_cmd(self):
+        if self._connection_info.password:
+            try:
+                p = subprocess.Popen(["sshpass"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                p.communicate()
+            except OSError:
+                raise AnsibleError("to use the 'ssh' connection type with passwords, you must install the sshpass program")
+            (self.rfd, self.wfd) = os.pipe()
+            return ["sshpass", "-d%d" % self.rfd]
+        return []
+
+    def _send_password(self):
+        if self._connection_info.password:
+            os.close(self.rfd)
+            os.write(self.wfd, "%s\n" % self._connection_info.password)
+            os.close(self.wfd)
+
+    def _communicate(self, p, stdin, indata, su=False, sudoable=False, prompt=None):
+        fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK)
+        fcntl.fcntl(p.stderr, fcntl.F_SETFL, fcntl.fcntl(p.stderr, fcntl.F_GETFL) & ~os.O_NONBLOCK)
+        # We can't use p.communicate here because the ControlMaster may have stdout open as well
+        stdout = ''
+        stderr = ''
+        rpipes = [p.stdout, p.stderr]
+        if indata:
+            try:
+                stdin.write(indata)
+                stdin.close()
+            except:
+                raise AnsibleConnectionFailure('SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh')
+        # Read stdout/stderr from process
+        while True:
+            rfd, wfd, efd = select.select(rpipes, [], rpipes, 1)
+
+            # FIXME: su/sudo stuff
+            # fail early if the sudo/su password is wrong
+            #if self.runner.sudo and sudoable:
+            #    if self.runner.sudo_pass:
+            #        incorrect_password = gettext.dgettext(
+            #            "sudo", "Sorry, try again.")
+            #        if stdout.endswith("%s\r\n%s" % (incorrect_password,
+            #                                         prompt)):
+            #            raise AnsibleError('Incorrect sudo password')
+            #
+            #    if stdout.endswith(prompt):
+            #        raise AnsibleError('Missing sudo password')
+            #
+            #if self.runner.su and su and self.runner.su_pass:
+            #    incorrect_password = gettext.dgettext(
+            #        "su", "Sorry")
+            #    if stdout.endswith("%s\r\n%s" % (incorrect_password, prompt)):
+            #        raise AnsibleError('Incorrect su password')
+
+            if p.stdout in rfd:
+                dat = os.read(p.stdout.fileno(), 9000)
+                stdout += dat
+                if dat == '':
+                    rpipes.remove(p.stdout)
+            if p.stderr in rfd:
+                dat = os.read(p.stderr.fileno(), 9000)
+                stderr += dat
+                if dat == '':
+                    rpipes.remove(p.stderr)
+            # only break out if no pipes are left to read or
+            # the pipes are completely read and
+            # the process is terminated
+            if (not rpipes or not rfd) and p.poll() is not None:
+                break
+            # No pipes are left to read but process is not yet terminated
+            # Only then it is safe to wait for the process to be finished
+            # NOTE: Actually p.poll() is always None here if rpipes is empty
+            elif not rpipes and p.poll() == None:
+                p.wait()
+                # The process is terminated. Since no pipes to read from are
+                # left, there is no need to call select() again.
+                break
+        # close stdin after process is terminated and stdout/stderr are read
+        # completely (see also issue #848)
+        stdin.close()
+        return (p.returncode, stdout, stderr)
+
+    def not_in_host_file(self, host):
+        if 'USER' in os.environ:
+            user_host_file = os.path.expandvars("~${USER}/.ssh/known_hosts")
+        else:
+            user_host_file = "~/.ssh/known_hosts"
+        user_host_file = os.path.expanduser(user_host_file)
+        
+        host_file_list = []
+        host_file_list.append(user_host_file)
+        host_file_list.append("/etc/ssh/ssh_known_hosts")
+        host_file_list.append("/etc/ssh/ssh_known_hosts2")
+        
+        hfiles_not_found = 0
+        for hf in host_file_list:
+            if not os.path.exists(hf):
+                hfiles_not_found += 1
+                continue
+            try:
+                host_fh = open(hf)
+            except IOError, e:
+                hfiles_not_found += 1
+                continue
+            else:
+                data = host_fh.read()
+                host_fh.close()
+                
+            for line in data.split("\n"):
+                if line is None or " " not in line:
+                    continue
+                tokens = line.split()
+                if tokens[0].find(self.HASHED_KEY_MAGIC) == 0:
+                    # this is a hashed known host entry
+                    try:
+                        (kn_salt,kn_host) = tokens[0][len(self.HASHED_KEY_MAGIC):].split("|",2)
+                        hash = hmac.new(kn_salt.decode('base64'), digestmod=sha1)
+                        hash.update(host)
+                        if hash.digest() == kn_host.decode('base64'):
+                            return False
+                    except:
+                        # invalid hashed host key, skip it
+                        continue
+                else:
+                    # standard host file entry
+                    if host in tokens[0]:
+                        return False
+
+        if (hfiles_not_found == len(host_file_list)):
+            self._display.vvv("EXEC previous known host file not found for %s" % host)
+        return True
+
+    def exec_command(self, cmd, tmp_path, executable='/bin/sh', in_data=None, sudoable=False):
+        ''' run a command on the remote host '''
+
+        ssh_cmd = self._password_cmd()
+        ssh_cmd += ["ssh", "-C"]
+        if not in_data:
+            # we can only use tty when we are not pipelining the modules. piping data into /usr/bin/python
+            # inside a tty automatically invokes the python interactive-mode but the modules are not
+            # compatible with the interactive-mode ("unexpected indent" mainly because of empty lines)
+            ssh_cmd += ["-tt"]
+        # FIXME: verbosity needs to move, most likely into connection info or
+        #        whatever other context we pass around instead of runner objects
+        #if utils.VERBOSITY > 3:
+        #    ssh_cmd += ["-vvv"]
+        #else:
+        #    ssh_cmd += ["-q"]
+        ssh_cmd += ["-q"]
+        ssh_cmd += self._common_args
+
+        #if self._ipv6:
+        #    ssh_cmd += ['-6']
+        ssh_cmd += [self._host.ipv4_address]
+
+        if not (self._connection_info.sudo or self._connection_info.su) or not sudoable:
+            prompt = None
+            if executable:
+                ssh_cmd.append(executable + ' -c ' + pipes.quote(cmd))
+            else:
+                ssh_cmd.append(cmd)
+        elif self._connection_info.su and self._connection_info.su_user:
+            su_cmd, prompt, success_key = self._connection_info.make_su_cmd(executable, cmd)
+            ssh_cmd.append(su_cmd)
+        else:
+            # FIXME: hard-coded sudo_exe here
+            sudo_cmd, prompt, success_key = self._connection_info.make_sudo_cmd('/usr/bin/sudo', executable, cmd)
+            ssh_cmd.append(sudo_cmd)
+
+        self._display.vvv("EXEC %s" % ' '.join(ssh_cmd), host=self._host)
+
+        not_in_host_file = self.not_in_host_file(self._host.get_name())
+
+        # FIXME: move the locations of these lock files, same as init above
+        #if C.HOST_KEY_CHECKING and not_in_host_file:
+        #    # lock around the initial SSH connectivity so the user prompt about whether to add 
+        #    # the host to known hosts is not intermingled with multiprocess output.
+        #    fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX)
+        #    fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_EX)
+
+        # create process
+        (p, stdin) = self._run(ssh_cmd, in_data)
+
+        self._send_password()
+
+        no_prompt_out = ''
+        no_prompt_err = ''
+        # FIXME: su/sudo stuff
+        #if (self.runner.sudo and sudoable and self.runner.sudo_pass) or \
+        #        (self.runner.su and su and self.runner.su_pass):
+        #    # several cases are handled for sudo privileges with password
+        #    # * NOPASSWD (tty & no-tty): detect success_key on stdout
+        #    # * without NOPASSWD:
+        #    #   * detect prompt on stdout (tty)
+        #    #   * detect prompt on stderr (no-tty)
+        #    fcntl.fcntl(p.stdout, fcntl.F_SETFL,
+        #                fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
+        #    fcntl.fcntl(p.stderr, fcntl.F_SETFL,
+        #                fcntl.fcntl(p.stderr, fcntl.F_GETFL) | os.O_NONBLOCK)
+        #    sudo_output = ''
+        #    sudo_errput = ''
+        #
+        #    while True:
+        #        if success_key in sudo_output or \
+        #            (self.runner.sudo_pass and sudo_output.endswith(prompt)) or \
+        #            (self.runner.su_pass and utils.su_prompts.check_su_prompt(sudo_output)):
+        #            break
+        self._display.vvv("EXEC %s" % ' '.join(ssh_cmd), host=self._host)
+
+        not_in_host_file = self.not_in_host_file(self._host.get_name())
+
+        # FIXME: file locations
+        #if C.HOST_KEY_CHECKING and not_in_host_file:
+        #    # lock around the initial SSH connectivity so the user prompt about whether to add 
+        #    # the host to known hosts is not intermingled with multiprocess output.
+        #    fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX)
+        #    fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_EX)
+
+        # create process
+        (p, stdin) = self._run(ssh_cmd, in_data)
+
+        self._send_password()
+
+        no_prompt_out = ''
+        no_prompt_err = ''
+        # FIXME: su/sudo stuff
+        #if (self.runner.sudo and sudoable and self.runner.sudo_pass) or \
+        #        (self.runner.su and su and self.runner.su_pass):
+        #    # several cases are handled for sudo privileges with password
+        #    # * NOPASSWD (tty & no-tty): detect success_key on stdout
+        #    # * without NOPASSWD:
+        #    #   * detect prompt on stdout (tty)
+        #    #   * detect prompt on stderr (no-tty)
+        #    fcntl.fcntl(p.stdout, fcntl.F_SETFL,
+        #                fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
+        #    fcntl.fcntl(p.stderr, fcntl.F_SETFL,
+        #                fcntl.fcntl(p.stderr, fcntl.F_GETFL) | os.O_NONBLOCK)
+        #    sudo_output = ''
+        #    sudo_errput = ''
+        #
+        #    while True:
+        #        if success_key in sudo_output or \
+        #            (self.runner.sudo_pass and sudo_output.endswith(prompt)) or \
+        #            (self.runner.su_pass and utils.su_prompts.check_su_prompt(sudo_output)):
+        #            break
+        #
+        #        rfd, wfd, efd = select.select([p.stdout, p.stderr], [],
+        #                                      [p.stdout], self.runner.timeout)
+        #        if p.stderr in rfd:
+        #            chunk = p.stderr.read()
+        #            if not chunk:
+        #                raise AnsibleError('ssh connection closed waiting for sudo or su password prompt')
+        #            sudo_errput += chunk
+        #            incorrect_password = gettext.dgettext(
+        #                "sudo", "Sorry, try again.")
+        #            if sudo_errput.strip().endswith("%s%s" % (prompt, incorrect_password)):
+        #                raise AnsibleError('Incorrect sudo password')
+        #            elif sudo_errput.endswith(prompt):
+        #                stdin.write(self.runner.sudo_pass + '\n')
+        #
+        #        if p.stdout in rfd:
+        #            chunk = p.stdout.read()
+        #            if not chunk:
+        #                raise AnsibleError('ssh connection closed waiting for sudo or su password prompt')
+        #            sudo_output += chunk
+        #
+        #        if not rfd:
+        #            # timeout. wrap up process communication
+        #            stdout = p.communicate()
+        #            raise AnsibleError('ssh connection error waiting for sudo or su password prompt')
+        #
+        #    if success_key not in sudo_output:
+        #        if sudoable:
+        #            stdin.write(self.runner.sudo_pass + '\n')
+        #        elif su:
+        #            stdin.write(self.runner.su_pass + '\n')
+        #    else:
+        #        no_prompt_out += sudo_output
+        #        no_prompt_err += sudo_errput
+
+        #(returncode, stdout, stderr) = self._communicate(p, stdin, in_data, su=su, sudoable=sudoable, prompt=prompt)
+        (returncode, stdout, stderr) = self._communicate(p, stdin, in_data, prompt=prompt)
+        
+        #if C.HOST_KEY_CHECKING and not_in_host_file:
+        #    # lock around the initial SSH connectivity so the user prompt about whether to add 
+        #    # the host to known hosts is not intermingled with multiprocess output.
+        #    fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_UN)
+        #    fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_UN)
+        controlpersisterror = 'Bad configuration option: ControlPersist' in stderr or 'unknown configuration option: ControlPersist' in stderr
+
+        if C.HOST_KEY_CHECKING:
+            if ssh_cmd[0] == "sshpass" and p.returncode == 6:
+                raise AnsibleError('Using a SSH password instead of a key is not possible because Host Key checking is enabled and sshpass does not support this.  Please add this host\'s fingerprint to your known_hosts file to manage this host.')
+
+        if p.returncode != 0 and controlpersisterror:
+            raise AnsibleError('using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" (or ssh_args in [ssh_connection] section of the config file) before running again')
+        # FIXME: module name isn't in runner
+        #if p.returncode == 255 and (in_data or self.runner.module_name == 'raw'):
+        if p.returncode == 255 and in_data:
+            raise AnsibleConnectionFailure('SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh')
+
+        return (p.returncode, '', no_prompt_out + stdout, no_prompt_err + stderr)
+
+    def put_file(self, in_path, out_path):
+        ''' transfer a file from local to remote '''
+        self._display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._host)
+        if not os.path.exists(in_path):
+            raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+        cmd = self._password_cmd()
+
+        # FIXME: make a function, used in all 3 methods EXEC/PUT/FETCH
+        host = self._host.ipv4_address
+        #if self._ipv6:
+        #    host = '[%s]' % host
+
+        if C.DEFAULT_SCP_IF_SSH:
+            cmd += ["scp"] + self._common_args
+            cmd += [in_path,host + ":" + pipes.quote(out_path)]
+            indata = None
+        else:
+            cmd += ["sftp"] + self._common_args + [host]
+            indata = "put %s %s\n" % (pipes.quote(in_path), pipes.quote(out_path))
+
+        (p, stdin) = self._run(cmd, indata)
+
+        self._send_password()
+
+        (returncode, stdout, stderr) = self._communicate(p, stdin, indata)
+
+        if returncode != 0:
+            raise AnsibleError("failed to transfer file to %s:\n%s\n%s" % (out_path, stdout, stderr))
+
+    def fetch_file(self, in_path, out_path):
+        ''' fetch a file from remote to local '''
+        self._display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self._host)
+        cmd = self._password_cmd()
+
+        # FIXME: make a function, used in all 3 methods EXEC/PUT/FETCH
+        host = self._host.ipv4_address
+        #if self._ipv6:
+        #    host = '[%s]' % self._host
+
+        if C.DEFAULT_SCP_IF_SSH:
+            cmd += ["scp"] + self._common_args
+            cmd += [host + ":" + in_path, out_path]
+            indata = None
+        else:
+            cmd += ["sftp"] + self._common_args + [host]
+            indata = "get %s %s\n" % (in_path, out_path)
+
+        p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self._send_password()
+        stdout, stderr = p.communicate(indata)
+
+        if p.returncode != 0:
+            raise AnsibleError("failed to transfer file from %s:\n%s\n%s" % (in_path, stdout, stderr))
+
+    def close(self):
+        ''' not applicable since we're executing openssh binaries '''
+        pass
+

v2/ansible/plugins/connections/winrm.py

@@ -0,0 +1,258 @@
+# (c) 2014, Chris Church <chris@ninemoreminutes.com>
+#
+# This file is part of Ansible.
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import absolute_import
+
+import base64
+import hashlib
+import imp
+import os
+import re
+import shlex
+import traceback
+import urlparse
+from ansible import errors
+from ansible import utils
+from ansible.callbacks import vvv, vvvv, verbose
+from ansible.runner.shell_plugins import powershell
+
+try:
+    from winrm import Response
+    from winrm.exceptions import WinRMTransportError
+    from winrm.protocol import Protocol
+except ImportError:
+    raise errors.AnsibleError("winrm is not installed")
+
+_winrm_cache = {
+    # 'user:pwhash@host:port': <protocol instance>
+}
+
+def vvvvv(msg, host=None):
+    verbose(msg, host=host, caplevel=4)
+
+class Connection(object):
+    '''WinRM connections over HTTP/HTTPS.'''
+
+    def __init__(self,  runner, host, port, user, password, *args, **kwargs):
+        self.runner = runner
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+        self.has_pipelining = False
+        self.default_shell = 'powershell'
+        self.default_suffixes = ['.ps1', '']
+        self.protocol = None
+        self.shell_id = None
+        self.delegate = None
+
+    def _winrm_connect(self):
+        '''
+        Establish a WinRM connection over HTTP/HTTPS.
+        '''
+        port = self.port or 5986
+        vvv("ESTABLISH WINRM CONNECTION FOR USER: %s on PORT %s TO %s" % \
+            (self.user, port, self.host), host=self.host)
+        netloc = '%s:%d' % (self.host, port)
+        cache_key = '%s:%s@%s:%d' % (self.user, hashlib.md5(self.password).hexdigest(), self.host, port)
+        if cache_key in _winrm_cache:
+            vvvv('WINRM REUSE EXISTING CONNECTION: %s' % cache_key, host=self.host)
+            return _winrm_cache[cache_key]
+        transport_schemes = [('plaintext', 'https'), ('plaintext', 'http')] # FIXME: ssl/kerberos
+        if port == 5985:
+            transport_schemes = reversed(transport_schemes)
+        exc = None
+        for transport, scheme in transport_schemes:
+            endpoint = urlparse.urlunsplit((scheme, netloc, '/wsman', '', ''))
+            vvvv('WINRM CONNECT: transport=%s endpoint=%s' % (transport, endpoint),
+                 host=self.host)
+            protocol = Protocol(endpoint, transport=transport,
+                                username=self.user, password=self.password)
+            try:
+                protocol.send_message('')
+                _winrm_cache[cache_key] = protocol
+                return protocol
+            except WinRMTransportError, exc:
+                err_msg = str(exc)
+                if re.search(r'Operation\s+?timed\s+?out', err_msg, re.I):
+                    raise errors.AnsibleError("the connection attempt timed out")
+                m = re.search(r'Code\s+?(\d{3})', err_msg)
+                if m:
+                    code = int(m.groups()[0])
+                    if code == 401:
+                        raise errors.AnsibleError("the username/password specified for this server was incorrect")
+                    elif code == 411:
+                        _winrm_cache[cache_key] = protocol
+                        return protocol
+                vvvv('WINRM CONNECTION ERROR: %s' % err_msg, host=self.host)
+                continue
+        if exc:
+            raise errors.AnsibleError(str(exc))
+
+    def _winrm_exec(self, command, args=(), from_exec=False):
+        if from_exec:
+            vvvv("WINRM EXEC %r %r" % (command, args), host=self.host)
+        else:
+            vvvvv("WINRM EXEC %r %r" % (command, args), host=self.host)
+        if not self.protocol:
+            self.protocol = self._winrm_connect()
+        if not self.shell_id:
+            self.shell_id = self.protocol.open_shell()
+        command_id = None
+        try:
+            command_id = self.protocol.run_command(self.shell_id, command, args)
+            response = Response(self.protocol.get_command_output(self.shell_id, command_id))
+            if from_exec:
+                vvvv('WINRM RESULT %r' % response, host=self.host)
+            else:
+                vvvvv('WINRM RESULT %r' % response, host=self.host)
+            vvvvv('WINRM STDOUT %s' % response.std_out, host=self.host)
+            vvvvv('WINRM STDERR %s' % response.std_err, host=self.host)
+            return response
+        finally:
+            if command_id:
+                self.protocol.cleanup_command(self.shell_id, command_id)
+
+    def connect(self):
+        if not self.protocol:
+            self.protocol = self._winrm_connect()
+        return self
+
+    def exec_command(self, cmd, tmp_path, sudo_user=None, sudoable=False, executable=None, in_data=None, su=None, su_user=None):
+        cmd = cmd.encode('utf-8')
+        cmd_parts = shlex.split(cmd, posix=False)
+        if '-EncodedCommand' in cmd_parts:
+            encoded_cmd = cmd_parts[cmd_parts.index('-EncodedCommand') + 1]
+            decoded_cmd = base64.b64decode(encoded_cmd)
+            vvv("EXEC %s" % decoded_cmd, host=self.host)
+        else:
+            vvv("EXEC %s" % cmd, host=self.host)
+        # For script/raw support.
+        if cmd_parts and cmd_parts[0].lower().endswith('.ps1'):
+            script = powershell._build_file_cmd(cmd_parts)
+            cmd_parts = powershell._encode_script(script, as_list=True)
+        try:
+            result = self._winrm_exec(cmd_parts[0], cmd_parts[1:], from_exec=True)
+        except Exception, e:
+            traceback.print_exc()
+            raise errors.AnsibleError("failed to exec cmd %s" % cmd)
+        return (result.status_code, '', result.std_out.encode('utf-8'), result.std_err.encode('utf-8'))
+
+    def put_file(self, in_path, out_path):
+        vvv("PUT %s TO %s" % (in_path, out_path), host=self.host)
+        if not os.path.exists(in_path):
+            raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+        with open(in_path) as in_file:
+            in_size = os.path.getsize(in_path)
+            script_template = '''
+                $s = [System.IO.File]::OpenWrite("%s");
+                [void]$s.Seek(%d, [System.IO.SeekOrigin]::Begin);
+                $b = [System.Convert]::FromBase64String("%s");
+                [void]$s.Write($b, 0, $b.length);
+                [void]$s.SetLength(%d);
+                [void]$s.Close();
+            '''
+            # Determine max size of data we can pass per command.
+            script = script_template % (powershell._escape(out_path), in_size, '', in_size)
+            cmd = powershell._encode_script(script)
+            # Encode script with no data, subtract its length from 8190 (max
+            # windows command length), divide by 2.67 (UTF16LE base64 command
+            # encoding), then by 1.35 again (data base64 encoding).
+            buffer_size = int(((8190 - len(cmd)) / 2.67) / 1.35)
+            for offset in xrange(0, in_size, buffer_size):
+                try:
+                    out_data = in_file.read(buffer_size)
+                    if offset == 0:
+                        if out_data.lower().startswith('#!powershell') and not out_path.lower().endswith('.ps1'):
+                            out_path = out_path + '.ps1'
+                    b64_data = base64.b64encode(out_data)
+                    script = script_template % (powershell._escape(out_path), offset, b64_data, in_size)
+                    vvvv("WINRM PUT %s to %s (offset=%d size=%d)" % (in_path, out_path, offset, len(out_data)), host=self.host)
+                    cmd_parts = powershell._encode_script(script, as_list=True)
+                    result = self._winrm_exec(cmd_parts[0], cmd_parts[1:])
+                    if result.status_code != 0:
+                        raise IOError(result.std_err.encode('utf-8'))
+                except Exception:
+                    traceback.print_exc()
+                    raise errors.AnsibleError("failed to transfer file to %s" % out_path)
+
+    def fetch_file(self, in_path, out_path):
+        out_path = out_path.replace('\\', '/')
+        vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host)
+        buffer_size = 2**20 # 1MB chunks
+        if not os.path.exists(os.path.dirname(out_path)):
+            os.makedirs(os.path.dirname(out_path))
+        out_file = None
+        try:
+            offset = 0
+            while True:
+                try:
+                    script = '''
+                        If (Test-Path -PathType Leaf "%(path)s")
+                        {
+                            $stream = [System.IO.File]::OpenRead("%(path)s");
+                            $stream.Seek(%(offset)d, [System.IO.SeekOrigin]::Begin) | Out-Null;
+                            $buffer = New-Object Byte[] %(buffer_size)d;
+                            $bytesRead = $stream.Read($buffer, 0, %(buffer_size)d);
+                            $bytes = $buffer[0..($bytesRead-1)];
+                            [System.Convert]::ToBase64String($bytes);
+                            $stream.Close() | Out-Null;
+                        }
+                        ElseIf (Test-Path -PathType Container "%(path)s")
+                        {
+                            Write-Host "[DIR]";
+                        }
+                        Else
+                        {
+                            Write-Error "%(path)s does not exist";
+                            Exit 1;
+                        }
+                    ''' % dict(buffer_size=buffer_size, path=powershell._escape(in_path), offset=offset)
+                    vvvv("WINRM FETCH %s to %s (offset=%d)" % (in_path, out_path, offset), host=self.host)
+                    cmd_parts = powershell._encode_script(script, as_list=True)
+                    result = self._winrm_exec(cmd_parts[0], cmd_parts[1:])
+                    if result.status_code != 0:
+                        raise IOError(result.std_err.encode('utf-8'))
+                    if result.std_out.strip() == '[DIR]':
+                        data = None
+                    else:
+                        data = base64.b64decode(result.std_out.strip())
+                    if data is None:
+                        if not os.path.exists(out_path):
+                            os.makedirs(out_path)
+                        break
+                    else:
+                        if not out_file:
+                            # If out_path is a directory and we're expecting a file, bail out now.
+                            if os.path.isdir(out_path):
+                                break
+                            out_file = open(out_path, 'wb')
+                        out_file.write(data)
+                        if len(data) < buffer_size:
+                            break
+                        offset += len(data)
+                except Exception:
+                    traceback.print_exc()
+                    raise errors.AnsibleError("failed to transfer file to %s" % out_path)
+        finally:
+            if out_file:
+                out_file.close()
+
+    def close(self):
+        if self.protocol and self.shell_id:
+            self.protocol.close_shell(self.shell_id)
+            self.shell_id = None

v2/ansible/plugins/filter/core.py

@@ -0,0 +1,323 @@
+# (c) 2012, Jeroen Hoekx <jeroen@hoekx.be>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+import json
+import os.path
+import yaml
+import types
+import pipes
+import glob
+import re
+import collections
+import operator as py_operator
+from distutils.version import LooseVersion, StrictVersion
+from random import SystemRandom, shuffle
+from jinja2.filters import environmentfilter
+
+from ansible.errors import *
+from ansible.utils.hashing import md5s, checksum_s
+
+def to_nice_yaml(*a, **kw):
+    '''Make verbose, human readable yaml'''
+    return yaml.safe_dump(*a, indent=4, allow_unicode=True, default_flow_style=False, **kw)
+
+def to_json(a, *args, **kw):
+    ''' Convert the value to JSON '''
+    return json.dumps(a, *args, **kw)
+
+def to_nice_json(a, *args, **kw):
+    '''Make verbose, human readable JSON'''
+    return json.dumps(a, indent=4, sort_keys=True, *args, **kw)
+
+def failed(*a, **kw):
+    ''' Test if task result yields failed '''
+    item = a[0]
+    if type(item) != dict:
+        raise errors.AnsibleFilterError("|failed expects a dictionary")
+    rc = item.get('rc',0)
+    failed = item.get('failed',False)
+    if rc != 0 or failed:
+        return True
+    else:
+        return False
+
+def success(*a, **kw):
+    ''' Test if task result yields success '''
+    return not failed(*a, **kw)
+
+def changed(*a, **kw):
+    ''' Test if task result yields changed '''
+    item = a[0]
+    if type(item) != dict:
+        raise errors.AnsibleFilterError("|changed expects a dictionary")
+    if not 'changed' in item:
+        changed = False
+        if ('results' in item    # some modules return a 'results' key
+                and type(item['results']) == list
+                and type(item['results'][0]) == dict):
+            for result in item['results']:
+                changed = changed or result.get('changed', False)
+    else:
+        changed = item.get('changed', False)
+    return changed
+
+def skipped(*a, **kw):
+    ''' Test if task result yields skipped '''
+    item = a[0]
+    if type(item) != dict:
+        raise errors.AnsibleFilterError("|skipped expects a dictionary")
+    skipped = item.get('skipped', False)
+    return skipped
+
+def mandatory(a):
+    ''' Make a variable mandatory '''
+    try:
+        a
+    except NameError:
+        raise errors.AnsibleFilterError('Mandatory variable not defined.')
+    else:
+        return a
+
+def bool(a):
+    ''' return a bool for the arg '''
+    if a is None or type(a) == bool:
+        return a
+    if type(a) in types.StringTypes:
+        a = a.lower()
+    if a in ['yes', 'on', '1', 'true', 1]:
+        return True
+    else:
+        return False
+
+def quote(a):
+    ''' return its argument quoted for shell usage '''
+    return pipes.quote(a)
+
+def fileglob(pathname):
+    ''' return list of matched files for glob '''
+    return glob.glob(pathname)
+
+def regex(value='', pattern='', ignorecase=False, match_type='search'):
+    ''' Expose `re` as a boolean filter using the `search` method by default.
+        This is likely only useful for `search` and `match` which already
+        have their own filters.
+    '''
+    if ignorecase:
+        flags = re.I
+    else:
+        flags = 0
+    _re = re.compile(pattern, flags=flags)
+    _bool = __builtins__.get('bool')
+    return _bool(getattr(_re, match_type, 'search')(value))
+
+def match(value, pattern='', ignorecase=False):
+    ''' Perform a `re.match` returning a boolean '''
+    return regex(value, pattern, ignorecase, 'match')
+
+def search(value, pattern='', ignorecase=False):
+    ''' Perform a `re.search` returning a boolean '''
+    return regex(value, pattern, ignorecase, 'search')
+
+def regex_replace(value='', pattern='', replacement='', ignorecase=False):
+    ''' Perform a `re.sub` returning a string '''
+
+    if not isinstance(value, basestring):
+        value = str(value)
+
+    if ignorecase:
+        flags = re.I
+    else:
+        flags = 0
+    _re = re.compile(pattern, flags=flags)
+    return _re.sub(replacement, value)
+
+def unique(a):
+    if isinstance(a,collections.Hashable):
+        c = set(a)
+    else:
+        c = []
+        for x in a:
+            if x not in c:
+                c.append(x)
+    return c
+
+def intersect(a, b):
+    if isinstance(a,collections.Hashable) and isinstance(b,collections.Hashable):
+        c = set(a) & set(b)
+    else:
+        c = unique(filter(lambda x: x in b, a))
+    return c
+
+def difference(a, b):
+    if isinstance(a,collections.Hashable) and isinstance(b,collections.Hashable):
+        c = set(a) - set(b)
+    else:
+        c = unique(filter(lambda x: x not in b, a))
+    return c
+
+def symmetric_difference(a, b):
+    if isinstance(a,collections.Hashable) and isinstance(b,collections.Hashable):
+        c = set(a) ^ set(b)
+    else:
+        c = unique(filter(lambda x: x not in intersect(a,b), union(a,b)))
+    return c
+
+def union(a, b):
+    if isinstance(a,collections.Hashable) and isinstance(b,collections.Hashable):
+        c = set(a) | set(b)
+    else:
+        c = unique(a + b)
+    return c
+
+def min(a):
+    _min = __builtins__.get('min')
+    return _min(a);
+
+def max(a):
+    _max = __builtins__.get('max')
+    return _max(a);
+
+def version_compare(value, version, operator='eq', strict=False):
+    ''' Perform a version comparison on a value '''
+    op_map = {
+        '==': 'eq', '=':  'eq', 'eq': 'eq',
+        '<':  'lt', 'lt': 'lt',
+        '<=': 'le', 'le': 'le',
+        '>':  'gt', 'gt': 'gt',
+        '>=': 'ge', 'ge': 'ge',
+        '!=': 'ne', '<>': 'ne', 'ne': 'ne'
+    }
+
+    if strict:
+        Version = StrictVersion
+    else:
+        Version = LooseVersion
+
+    if operator in op_map:
+        operator = op_map[operator]
+    else:
+        raise errors.AnsibleFilterError('Invalid operator type')
+
+    try:
+        method = getattr(py_operator, operator)
+        return method(Version(str(value)), Version(str(version)))
+    except Exception, e:
+        raise errors.AnsibleFilterError('Version comparison: %s' % e)
+
+@environmentfilter
+def rand(environment, end, start=None, step=None):
+    r = SystemRandom()
+    if isinstance(end, (int, long)):
+        if not start:
+            start = 0
+        if not step:
+            step = 1
+        return r.randrange(start, end, step)
+    elif hasattr(end, '__iter__'):
+        if start or step:
+            raise errors.AnsibleFilterError('start and step can only be used with integer values')
+        return r.choice(end)
+    else:
+        raise errors.AnsibleFilterError('random can only be used on sequences and integers')
+
+def randomize_list(mylist):
+    try:
+        mylist = list(mylist)
+        shuffle(mylist)
+    except:
+        pass
+    return mylist
+
+class FilterModule(object):
+    ''' Ansible core jinja2 filters '''
+
+    def filters(self):
+        return {
+            # base 64
+            'b64decode': base64.b64decode,
+            'b64encode': base64.b64encode,
+
+            # json
+            'to_json': to_json,
+            'to_nice_json': to_nice_json,
+            'from_json': json.loads,
+
+            # yaml
+            'to_yaml': yaml.safe_dump,
+            'to_nice_yaml': to_nice_yaml,
+            'from_yaml': yaml.safe_load,
+
+            # path
+            'basename': os.path.basename,
+            'dirname': os.path.dirname,
+            'expanduser': os.path.expanduser,
+            'realpath': os.path.realpath,
+            'relpath': os.path.relpath,
+
+            # failure testing
+            'failed'  : failed,
+            'success' : success,
+
+            # changed testing
+            'changed' : changed,
+
+            # skip testing
+            'skipped' : skipped,
+
+            # variable existence
+            'mandatory': mandatory,
+
+            # value as boolean
+            'bool': bool,
+
+            # quote string for shell usage
+            'quote': quote,
+
+            # hash filters
+            # md5 hex digest of string
+            'md5': md5s,
+            # sha1 hex digeset of string
+            'sha1': checksum_s,
+            # checksum of string as used by ansible for checksuming files
+            'checksum': checksum_s,
+
+            # file glob
+            'fileglob': fileglob,
+
+            # regex
+            'match': match,
+            'search': search,
+            'regex': regex,
+            'regex_replace': regex_replace,
+
+            # list
+            'unique' : unique,
+            'intersect': intersect,
+            'difference': difference,
+            'symmetric_difference': symmetric_difference,
+            'union': union,
+            'min' : min,
+            'max' : max,
+
+            # version comparison
+            'version_compare': version_compare,
+
+            # random stuff
+            'random': rand,
+            'shuffle': randomize_list,
+        }

v2/ansible/plugins/inventory/aggregate.py

@@ -1,61 +0,0 @@
-# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-#############################################
-
-# Make coding more python3-ish
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-import os
-
-from . import InventoryParser
-#from . ini import InventoryIniParser
-#from . script import InventoryScriptParser
-
-class InventoryAggregateParser(InventoryParser):
-
-    def __init__(self, inven_sources):
-        self.inven_source = inven_sources
-        self.hosts = dict()
-        self.groups = dict()
-
-    def reset_parser(self):
-        super(InventoryAggregateParser, self).reset_parser()
-
-    def parse(self, refresh=False):
-        # InventoryDirectoryParser is a InventoryAggregateParser so we avoid
-        # a circular import by importing here
-        from . directory import InventoryAggregateParser
-        if super(InventoryAggregateParser, self).parse(refresh):
-            return self.parsed
-
-        for entry in self.inven_sources:
-            if os.path.sep in entry:
-                # file or directory
-                if os.path.isdir(entry):
-                    parser = directory.InventoryDirectoryParser(filename=entry)
-                elif utils.is_executable(entry):
-                    parser = InventoryScriptParser(filename=entry)
-                else:
-                    parser = InventoryIniParser(filename=entry)
-            else:
-                # hostname
-                parser = HostnameParser(hostname=entry)
-            hosts, groups = parser.parse()
-            self._merge(self.hosts, hosts)
-            self._merge(self.groups, groups)

v2/ansible/plugins/inventory/ini.py

@@ -51,3 +51,10 @@ class InventoryIniParser(InventoryAggregateParser):
     def parse(self):
         return super(InventoryDirectoryParser, self).parse()
 
+    def _before_comment(self, msg):
+        ''' what's the part of a string before a comment? '''
+        msg = msg.replace("\#","**NOT_A_COMMENT**")
+        msg = msg.split("#")[0]
+        msg = msg.replace("**NOT_A_COMMENT**","#")
+        return msg
+

v2/ansible/plugins/lookup/csvfile.py

@@ -1,82 +0,0 @@
-# (c) 2013, Jan-Piet Mens <jpmens(at)gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from ansible import utils, errors
-import os
-import codecs
-import csv
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def read_csv(self, filename, key, delimiter, dflt=None, col=1):
-
-        try:
-            f = codecs.open(filename, 'r', encoding='utf-8')
-            creader = csv.reader(f, delimiter=delimiter)
-
-            for row in creader:
-                if row[0] == key:
-                    return row[int(col)]
-        except Exception, e:
-            raise errors.AnsibleError("csvfile: %s" % str(e))
-
-        return dflt
-
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-
-        if isinstance(terms, basestring):
-            terms = [ terms ]
-
-        ret = []
-        for term in terms:
-            params = term.split()
-            key = params[0]
-
-            paramvals = {
-                'file' : 'ansible.csv',
-                'default' : None,
-                'delimiter' : "TAB",
-                'col' : "1",          # column to return
-            }
-
-            # parameters specified?
-            try:
-                for param in params[1:]:
-                    name, value = param.split('=')
-                    assert(name in paramvals)
-                    paramvals[name] = value
-            except (ValueError, AssertionError), e:
-                raise errors.AnsibleError(e)
-
-            if paramvals['delimiter'] == 'TAB':
-                paramvals['delimiter'] = "\t"
-
-            path = utils.path_dwim(self.basedir, paramvals['file'])
-
-            var = self.read_csv(path, key, paramvals['delimiter'], paramvals['default'], paramvals['col'])
-            if var is not None:
-                if type(var) is list:
-                    for v in var:
-                        ret.append(v)
-                else:
-                    ret.append(var)
-        return ret

v2/ansible/plugins/lookup/dict.py

@@ -1,39 +0,0 @@
-# (c) 2014, Kent R. Spillner <kspillner@acm.org>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from ansible.utils import safe_eval
-import ansible.utils as utils
-import ansible.errors as errors
-
-def flatten_hash_to_list(terms):
-    ret = []
-    for key in terms:
-        ret.append({'key': key, 'value': terms[key]})
-    return ret
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def run(self, terms, inject=None, **kwargs):
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-
-        if not isinstance(terms, dict):
-            raise errors.AnsibleError("with_dict expects a dict")
-
-        return flatten_hash_to_list(terms)

v2/ansible/plugins/lookup/dnstxt.py

@@ -1,68 +0,0 @@
-# (c) 2012, Jan-Piet Mens <jpmens(at)gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from ansible import utils, errors
-import os
-HAVE_DNS=False
-try:
-    import dns.resolver
-    from dns.exception import DNSException
-    HAVE_DNS=True
-except ImportError:
-    pass
-
-# ==============================================================
-# DNSTXT: DNS TXT records
-#
-#       key=domainname
-# TODO: configurable resolver IPs
-# --------------------------------------------------------------
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-        if HAVE_DNS == False:
-            raise errors.AnsibleError("Can't LOOKUP(dnstxt): module dns.resolver is not installed")
-
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
-
-        if isinstance(terms, basestring):
-            terms = [ terms ]
-
-        ret = []
-        for term in terms:
-            domain = term.split()[0]
-            string = []
-            try:
-                answers = dns.resolver.query(domain, 'TXT')
-                for rdata in answers:
-                    s = rdata.to_text()
-                    string.append(s[1:-1])  # Strip outside quotes on TXT rdata
-
-            except dns.resolver.NXDOMAIN:
-                string = 'NXDOMAIN'
-            except dns.resolver.Timeout:
-                string = ''
-            except dns.exception.DNSException, e:
-                raise errors.AnsibleError("dns.resolver unhandled exception", e)
-
-            ret.append(''.join(string))
-        return ret

v2/ansible/plugins/lookup/env.py

@@ -1,41 +0,0 @@
-# (c) 2012, Jan-Piet Mens <jpmens(at)gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from ansible import utils, errors
-from ansible.utils import template
-import os
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def run(self, terms, inject=None, **kwargs):
-
-        try:
-            terms = template.template(self.basedir, terms, inject)
-        except Exception, e:
-            pass
-
-        if isinstance(terms, basestring):
-            terms = [ terms ]
-
-        ret = []
-        for term in terms:
-            var = term.split()[0]
-            ret.append(os.getenv(var, ''))
-        return ret

v2/ansible/plugins/lookup/etcd.py

@@ -1,78 +0,0 @@
-# (c) 2013, Jan-Piet Mens <jpmens(at)gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from ansible import utils
-import os
-import urllib2
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
-# this can be made configurable, not should not use ansible.cfg
-ANSIBLE_ETCD_URL = 'http://127.0.0.1:4001'
-if os.getenv('ANSIBLE_ETCD_URL') is not None:
-    ANSIBLE_ETCD_URL = os.environ['ANSIBLE_ETCD_URL']
-
-class etcd():
-    def __init__(self, url=ANSIBLE_ETCD_URL):
-        self.url = url
-        self.baseurl = '%s/v1/keys' % (self.url)
-
-    def get(self, key):
-        url = "%s/%s" % (self.baseurl, key)
-
-        data = None
-        value = ""
-        try:
-            r = urllib2.urlopen(url)
-            data = r.read()
-        except:
-            return value
-
-        try:
-            # {"action":"get","key":"/name","value":"Jane Jolie","index":5}
-            item = json.loads(data)
-            if 'value' in item:
-                value = item['value']
-            if 'errorCode' in item:
-                value = "ENOENT"
-        except:
-            raise
-            pass
-
-        return value
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-        self.etcd = etcd()
-
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-
-        if isinstance(terms, basestring):
-            terms = [ terms ]
-
-        ret = []
-        for term in terms:
-            key = term.split()[0]
-            value = self.etcd.get(key)
-            ret.append(value)
-        return ret

v2/ansible/plugins/lookup/file.py

@@ -1,59 +0,0 @@
-# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from ansible import utils, errors
-import os
-import codecs
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-        ret = []
-
-        # this can happen if the variable contains a string, strictly not desired for lookup
-        # plugins, but users may try it, so make it work.
-        if not isinstance(terms, list):
-            terms = [ terms ]
-
-        for term in terms:
-            basedir_path  = utils.path_dwim(self.basedir, term)
-            relative_path = None
-            playbook_path = None
-
-            # Special handling of the file lookup, used primarily when the
-            # lookup is done from a role. If the file isn't found in the
-            # basedir of the current file, use dwim_relative to look in the
-            # role/files/ directory, and finally the playbook directory
-            # itself (which will be relative to the current working dir)
-            if '_original_file' in inject:
-                relative_path = utils.path_dwim_relative(inject['_original_file'], 'files', term, self.basedir, check=False)
-            if 'playbook_dir' in inject:
-                playbook_path = os.path.join(inject['playbook_dir'], term)
-
-            for path in (basedir_path, relative_path, playbook_path):
-                if path and os.path.exists(path):
-                    ret.append(codecs.open(path, encoding="utf8").read().rstrip())
-                    break
-            else:
-                raise errors.AnsibleError("could not locate file in lookup: %s" % term)
-
-        return ret

v2/ansible/plugins/lookup/first_found.py

@@ -1,194 +0,0 @@
-# (c) 2013, seth vidal <skvidal@fedoraproject.org> red hat, inc
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-
-# take a list of files and (optionally) a list of paths
-# return the first existing file found in the paths
-# [file1, file2, file3], [path1, path2, path3]
-# search order is:
-# path1/file1
-# path1/file2
-# path1/file3
-# path2/file1
-# path2/file2
-# path2/file3
-# path3/file1
-# path3/file2
-# path3/file3
-
-# first file found with os.path.exists() is returned
-# no file matches raises ansibleerror
-# EXAMPLES
-#  - name: copy first existing file found to /some/file
-#    action: copy src=$item dest=/some/file
-#    with_first_found: 
-#     - files: foo ${inventory_hostname} bar
-#       paths: /tmp/production /tmp/staging
-
-# that will look for files in this order:
-# /tmp/production/foo
-#                 ${inventory_hostname}
-#                 bar
-# /tmp/staging/foo
-#              ${inventory_hostname}
-#              bar
-                  
-#  - name: copy first existing file found to /some/file
-#    action: copy src=$item dest=/some/file
-#    with_first_found: 
-#     - files: /some/place/foo ${inventory_hostname} /some/place/else
-
-#  that will look for files in this order:
-#  /some/place/foo
-#  $relative_path/${inventory_hostname}
-#  /some/place/else
-
-# example - including tasks:
-#  tasks:
-#  - include: $item
-#    with_first_found:
-#     - files: generic
-#       paths: tasks/staging tasks/production
-# this will include the tasks in the file generic where it is found first (staging or production)
-
-# example simple file lists
-#tasks:
-#- name: first found file
-#  action: copy src=$item dest=/etc/file.cfg
-#  with_first_found:
-#  - files: foo.${inventory_hostname} foo
-
-
-# example skipping if no matched files
-# First_found also offers the ability to control whether or not failing
-# to find a file returns an error or not
-#
-#- name: first found file - or skip
-#  action: copy src=$item dest=/etc/file.cfg
-#  with_first_found:
-#  - files: foo.${inventory_hostname}
-#    skip: true
-
-# example a role with default configuration and configuration per host
-# you can set multiple terms with their own files and paths to look through.
-# consider a role that sets some configuration per host falling back on a default config.
-#
-#- name: some configuration template
-#  template: src={{ item }} dest=/etc/file.cfg mode=0444 owner=root group=root
-#  with_first_found:
-#   - files:
-#      - ${inventory_hostname}/etc/file.cfg
-#     paths:
-#      - ../../../templates.overwrites
-#      - ../../../templates
-#   - files:
-#      - etc/file.cfg
-#     paths:
-#      - templates
-
-# the above will return an empty list if the files cannot be found at all
-# if skip is unspecificed or if it is set to false then it will return a list 
-# error which can be caught bye ignore_errors: true for that action.
-
-# finally - if you want you can use it, in place to replace first_available_file:
-# you simply cannot use the - files, path or skip options. simply replace
-# first_available_file with with_first_found and leave the file listing in place
-#
-#
-#  - name: with_first_found like first_available_file
-#    action: copy src=$item dest=/tmp/faftest
-#    with_first_found:
-#     - ../files/foo
-#     - ../files/bar
-#     - ../files/baz
-#    ignore_errors: true
-
-
-from ansible import utils, errors
-import os
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-
-        result = None
-        anydict = False
-        skip = False
-
-        for term in terms:
-            if isinstance(term, dict):
-                anydict = True
-
-        total_search = []
-        if anydict:
-            for term in terms:
-                if isinstance(term, dict):
-                    files = term.get('files', [])
-                    paths = term.get('paths', [])
-                    skip  = utils.boolean(term.get('skip', False))
-
-                    filelist = files
-                    if isinstance(files, basestring):
-                        files = files.replace(',', ' ')
-                        files = files.replace(';', ' ')
-                        filelist = files.split(' ')
-
-                    pathlist = paths
-                    if paths:
-                        if isinstance(paths, basestring):
-                            paths = paths.replace(',', ' ')
-                            paths = paths.replace(':', ' ')
-                            paths = paths.replace(';', ' ')
-                            pathlist = paths.split(' ')
-
-                    if not pathlist:
-                        total_search = filelist
-                    else:
-                        for path in pathlist:
-                            for fn in filelist:
-                                f = os.path.join(path, fn)
-                                total_search.append(f)
-                else:
-                    total_search.append(term)
-        else:
-            total_search = terms
-
-        for fn in total_search:
-            if inject and '_original_file' in inject:
-                # check the templates and vars directories too,
-                # if they exist
-                for roledir in ('templates', 'vars'):
-                    path = utils.path_dwim(os.path.join(self.basedir, '..', roledir), fn)
-                    if os.path.exists(path):
-                        return [path]
-            # if none of the above were found, just check the
-            # current filename against the basedir (this will already
-            # have ../files from runner, if it's a role task
-            path = utils.path_dwim(self.basedir, fn)
-            if os.path.exists(path):
-                return [path]
-        else:
-            if skip:
-                return []
-            else:
-                return [None]
-

v2/ansible/plugins/lookup/flattened.py

@@ -1,78 +0,0 @@
-# (c) 2013, Serge van Ginderachter <serge@vanginderachter.be>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-import ansible.utils as utils
-import ansible.errors as errors
-
-
-def check_list_of_one_list(term):
-    # make sure term is not a list of one (list of one..) item
-    # return the final non list item if so
-
-    if isinstance(term,list) and len(term) == 1:
-        term = term[0]
-        if isinstance(term,list):
-            term = check_list_of_one_list(term)
-
-    return term
-
-
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-
-    def flatten(self, terms, inject):
-
-        ret = []
-        for term in terms:
-            term = check_list_of_one_list(term)
-
-            if term == 'None' or term == 'null':
-                # ignore undefined items
-                break
-
-            if isinstance(term, basestring):
-                # convert a variable to a list
-                term2 = utils.listify_lookup_plugin_terms(term, self.basedir, inject)
-                # but avoid converting a plain string to a list of one string
-                if term2 != [ term ]:
-                    term = term2
-
-            if isinstance(term, list):
-                # if it's a list, check recursively for items that are a list
-                term = self.flatten(term, inject)
-                ret.extend(term)
-            else:   
-                ret.append(term)
-
-        return ret
-
-
-    def run(self, terms, inject=None, **kwargs):
-
-        # see if the string represents a list and convert to list if so
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-
-        if not isinstance(terms, list):
-            raise errors.AnsibleError("with_flattened expects a list")
-
-        ret = self.flatten(terms, inject)
-        return ret
-

v2/ansible/plugins/lookup/indexed_items.py

@@ -1,44 +0,0 @@
-# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from ansible.utils import safe_eval
-import ansible.utils as utils
-import ansible.errors as errors
-
-def flatten(terms):
-    ret = []
-    for term in terms:
-        if isinstance(term, list):
-            ret.extend(term)
-        else:
-            ret.append(term)
-    return ret
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def run(self, terms, inject=None, **kwargs):
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-
-        if not isinstance(terms, list):
-            raise errors.AnsibleError("with_indexed_items expects a list")
-
-        items = flatten(terms)
-        return zip(range(len(items)), items)
-

v2/ansible/plugins/lookup/inventory_hostnames.py

@@ -1,48 +0,0 @@
-# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-# (c) 2013, Steven Dossett <sdossett@panath.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from ansible.utils import safe_eval
-import ansible.utils as utils
-import ansible.errors as errors
-import ansible.inventory as inventory
-
-def flatten(terms):
-    ret = []
-    for term in terms:
-        if isinstance(term, list):
-            ret.extend(term)
-        else:
-            ret.append(term)
-    return ret
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-        if 'runner' in kwargs:
-            self.host_list = kwargs['runner'].inventory.host_list
-        else:
-            raise errors.AnsibleError("inventory_hostnames must be used as a loop. Example: \"with_inventory_hostnames: \'all\'\"")
-
-    def run(self, terms, inject=None, **kwargs):
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
-
-        if not isinstance(terms, list):
-            raise errors.AnsibleError("with_inventory_hostnames expects a list")
-        return flatten(inventory.Inventory(self.host_list).list_hosts(terms))
-

v2/ansible/plugins/lookup/items.py

@@ -15,9 +15,9 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
-from ansible.utils import safe_eval
-import ansible.utils as utils
-import ansible.errors as errors
+#from ansible.utils import safe_eval
+#import ansible.utils as utils
+#import ansible.errors as errors
 
 def flatten(terms):
     ret = []
@@ -34,10 +34,10 @@ class LookupModule(object):
         self.basedir = basedir
 
     def run(self, terms, inject=None, **kwargs):
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
-
-        if not isinstance(terms, list) and not isinstance(terms,set):
-            raise errors.AnsibleError("with_items expects a list or a set")
+        # FIXME: this function needs to be ported still, or something like it
+        #        where really the intention is just to template a bare variable
+        #        with the result being a list of terms
+        #terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
 
         return flatten(terms)
 

v2/ansible/plugins/lookup/lines.py

@@ -1,38 +0,0 @@
-# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-import subprocess
-from ansible import utils, errors
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
-
-        ret = []
-        for term in terms:
-            p = subprocess.Popen(term, cwd=self.basedir, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-            (stdout, stderr) = p.communicate()
-            if p.returncode == 0:
-                ret.extend(stdout.splitlines())
-            else:
-                raise errors.AnsibleError("lookup_plugin.lines(%s) returned %d" % (term, p.returncode))
-        return ret

v2/ansible/plugins/lookup/nested.py

@@ -1,73 +0,0 @@
-# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-import ansible.utils as utils
-from ansible.utils import safe_eval
-import ansible.errors as errors
-
-def flatten(terms):
-    ret = []
-    for term in terms:
-        if isinstance(term, list):
-            ret.extend(term)
-        elif isinstance(term, tuple):
-            ret.extend(term)
-        else:
-            ret.append(term)
-    return ret
-
-def combine(a,b):
-    results = []
-    for x in a:
-        for y in b:
-            results.append(flatten([x,y]))
-    return results
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def __lookup_injects(self, terms, inject):
-        results = []
-        for x in terms:
-            intermediate = utils.listify_lookup_plugin_terms(x, self.basedir, inject)
-            results.append(intermediate)
-        return results
-
-    def run(self, terms, inject=None, **kwargs):
-
-        # this code is common with 'items.py' consider moving to utils if we need it again
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-        terms = self.__lookup_injects(terms, inject)
-
-        my_list = terms[:]
-        my_list.reverse()
-        result = []
-        if len(my_list) == 0:
-            raise errors.AnsibleError("with_nested requires at least one element in the nested list")
-        result = my_list.pop()
-        while len(my_list) > 0:
-            result2 = combine(result, my_list.pop())
-            result  = result2
-        new_result = []
-        for x in result:
-            new_result.append(flatten(x))
-        return new_result
-
-

v2/ansible/plugins/lookup/password.py

@@ -1,129 +0,0 @@
-# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
-# (c) 2013, Javier Candeira <javier@candeira.com>
-# (c) 2013, Maykel Moya <mmoya@speedyrails.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from ansible import utils, errors
-import os
-import errno
-from string import ascii_letters, digits
-import string
-import random
-
-
-class LookupModule(object):
-
-    LENGTH = 20
-
-    def __init__(self, length=None, encrypt=None, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def random_salt(self):
-        salt_chars = ascii_letters + digits + './'
-        return utils.random_password(length=8, chars=salt_chars)
-
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
-
-        ret = []
-
-        for term in terms:
-            # you can't have escaped spaces in yor pathname
-            params = term.split()
-            relpath = params[0]
-
-            paramvals = {
-                'length': LookupModule.LENGTH,
-                'encrypt': None,
-                'chars': ['ascii_letters','digits',".,:-_"],
-            }
-
-            # get non-default parameters if specified
-            try:
-                for param in params[1:]:
-                    name, value = param.split('=')
-                    assert(name in paramvals)
-                    if name == 'length':
-                        paramvals[name] = int(value)
-                    elif name == 'chars':
-                        use_chars=[]
-                        if ",," in value: 
-                            use_chars.append(',')
-                        use_chars.extend(value.replace(',,',',').split(','))
-                        paramvals['chars'] = use_chars
-                    else:
-                        paramvals[name] = value
-            except (ValueError, AssertionError), e:
-                raise errors.AnsibleError(e)
-
-            length  = paramvals['length']
-            encrypt = paramvals['encrypt']
-            use_chars = paramvals['chars']
-
-            # get password or create it if file doesn't exist
-            path = utils.path_dwim(self.basedir, relpath)
-            if not os.path.exists(path):
-                pathdir = os.path.dirname(path)
-                if not os.path.isdir(pathdir):
-                    try:
-                        os.makedirs(pathdir, mode=0700)
-                    except OSError, e:
-                        raise errors.AnsibleError("cannot create the path for the password lookup: %s (error was %s)" % (pathdir, str(e)))
-
-                chars = "".join([getattr(string,c,c) for c in use_chars]).replace('"','').replace("'",'')
-                password = ''.join(random.choice(chars) for _ in range(length))
-
-                if encrypt is not None:
-                    salt = self.random_salt()
-                    content = '%s salt=%s' % (password, salt)
-                else:
-                    content = password
-                with open(path, 'w') as f:
-                    os.chmod(path, 0600)
-                    f.write(content + '\n')
-            else:
-                content = open(path).read().rstrip()
-                sep = content.find(' ')
-
-                if sep >= 0:
-                    password = content[:sep]
-                    salt = content[sep+1:].split('=')[1]
-                else:
-                    password = content
-                    salt = None
-
-                # crypt requested, add salt if missing
-                if (encrypt is not None and not salt):
-                    salt = self.random_salt()
-                    content = '%s salt=%s' % (password, salt)
-                    with open(path, 'w') as f:
-                        os.chmod(path, 0600)
-                        f.write(content + '\n')
-                # crypt not requested, remove salt if present
-                elif (encrypt is None and salt):
-                    with open(path, 'w') as f:
-                        os.chmod(path, 0600)
-                        f.write(password + '\n')
-
-            if encrypt:
-                password = utils.do_encrypt(password, encrypt, salt=salt)
-
-            ret.append(password)
-
-        return ret
-

v2/ansible/plugins/lookup/pipe.py

@@ -1,52 +0,0 @@
-# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-import subprocess
-from ansible import utils, errors
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
-
-        if isinstance(terms, basestring):
-            terms = [ terms ] 
-
-        ret = []
-        for term in terms:
-            '''
-            http://docs.python.org/2/library/subprocess.html#popen-constructor
-
-            The shell argument (which defaults to False) specifies whether to use the 
-            shell as the program to execute. If shell is True, it is recommended to pass 
-            args as a string rather than as a sequence
-
-            https://github.com/ansible/ansible/issues/6550
-            '''
-            term = str(term)
-
-            p = subprocess.Popen(term, cwd=self.basedir, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-            (stdout, stderr) = p.communicate()
-            if p.returncode == 0:
-                ret.append(stdout.decode("utf-8").rstrip())
-            else:
-                raise errors.AnsibleError("lookup_plugin.pipe(%s) returned %d" % (term, p.returncode))
-        return ret

v2/ansible/plugins/lookup/random_choice.py

@@ -1,41 +0,0 @@
-# (c) 2013, Michael DeHaan <michael.dehaan@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-import random
-from ansible import utils
-
-# useful for introducing chaos ... or just somewhat reasonably fair selection
-# amongst available mirrors
-#
-#    tasks:
-#        - debug: msg=$item
-#          with_random_choice:
-#             - one
-#             - two 
-#             - three
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
-
-        return [ random.choice(terms) ]
-

v2/ansible/plugins/lookup/redis_kv.py

@@ -1,72 +0,0 @@
-# (c) 2012, Jan-Piet Mens <jpmens(at)gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from ansible import utils, errors
-import os
-HAVE_REDIS=False
-try:
-    import redis        # https://github.com/andymccurdy/redis-py/
-    HAVE_REDIS=True
-except ImportError:
-    pass
-import re
-
-# ==============================================================
-# REDISGET: Obtain value from a GET on a Redis key. Terms
-# expected: 0 = URL, 1 = Key
-# URL may be empty, in which case redis://localhost:6379 assumed
-# --------------------------------------------------------------
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-        if HAVE_REDIS == False:
-            raise errors.AnsibleError("Can't LOOKUP(redis_kv): module redis is not installed")
-
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
-
-        ret = []
-        for term in terms:
-            (url,key) = term.split(',')
-            if url == "":
-                url = 'redis://localhost:6379'
-
-            # urlsplit on Python 2.6.1 is broken. Hmm. Probably also the reason
-            # Redis' from_url() doesn't work here.
-
-            p = '(?P<scheme>[^:]+)://?(?P<host>[^:/ ]+).?(?P<port>[0-9]*).*'
-
-            try:
-                m = re.search(p, url)
-                host = m.group('host')
-                port = int(m.group('port'))
-            except AttributeError:
-                raise errors.AnsibleError("Bad URI in redis lookup")
-
-            try:
-                conn = redis.Redis(host=host, port=port)
-                res = conn.get(key)
-                if res is None:
-                    res = ""
-                ret.append(res)
-            except:
-                ret.append("")  # connection failed or key not found
-        return ret

v2/ansible/plugins/lookup/sequence.py

@@ -1,204 +0,0 @@
-# (c) 2013, Jayson Vantuyl <jayson@aggressive.ly>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-from ansible.errors import AnsibleError
-import ansible.utils as utils
-from re import compile as re_compile, IGNORECASE
-
-# shortcut format
-NUM = "(0?x?[0-9a-f]+)"
-SHORTCUT = re_compile(
-    "^(" +        # Group 0
-    NUM +         # Group 1: Start
-    "-)?" +
-    NUM +         # Group 2: End
-    "(/" +        # Group 3
-    NUM +         # Group 4: Stride
-    ")?" +
-    "(:(.+))?$",  # Group 5, Group 6: Format String
-    IGNORECASE
-)
-
-
-class LookupModule(object):
-    """
-    sequence lookup module
-
-    Used to generate some sequence of items. Takes arguments in two forms.
-
-    The simple / shortcut form is:
-
-      [start-]end[/stride][:format]
-
-    As indicated by the brackets: start, stride, and format string are all
-    optional.  The format string is in the style of printf.  This can be used
-    to pad with zeros, format in hexadecimal, etc.  All of the numerical values
-    can be specified in octal (i.e. 0664) or hexadecimal (i.e. 0x3f8).
-    Negative numbers are not supported.
-
-    Some examples:
-
-      5 -> ["1","2","3","4","5"]
-      5-8 -> ["5", "6", "7", "8"]
-      2-10/2 -> ["2", "4", "6", "8", "10"]
-      4:host%02d -> ["host01","host02","host03","host04"]
-
-    The standard Ansible key-value form is accepted as well.  For example:
-
-      start=5 end=11 stride=2 format=0x%02x -> ["0x05","0x07","0x09","0x0a"]
-
-    This format takes an alternate form of "end" called "count", which counts
-    some number from the starting value.  For example:
-
-      count=5 -> ["1", "2", "3", "4", "5"]
-      start=0x0f00 count=4 format=%04x -> ["0f00", "0f01", "0f02", "0f03"]
-      start=0 count=5 stride=2 -> ["0", "2", "4", "6", "8"]
-      start=1 count=5 stride=2 -> ["1", "3", "5", "7", "9"]
-
-    The count option is mostly useful for avoiding off-by-one errors and errors
-    calculating the number of entries in a sequence when a stride is specified.
-    """
-
-    def __init__(self, basedir, **kwargs):
-        """absorb any keyword args"""
-        self.basedir = basedir
-
-    def reset(self):
-        """set sensible defaults"""
-        self.start = 1
-        self.count = None
-        self.end = None
-        self.stride = 1
-        self.format = "%d"
-
-    def parse_kv_args(self, args):
-        """parse key-value style arguments"""
-        for arg in ["start", "end", "count", "stride"]:
-            try:
-                arg_raw = args.pop(arg, None)
-                if arg_raw is None:
-                    continue
-                arg_cooked = int(arg_raw, 0)
-                setattr(self, arg, arg_cooked)
-            except ValueError:
-                raise AnsibleError(
-                    "can't parse arg %s=%r as integer"
-                        % (arg, arg_raw)
-                )
-            if 'format' in args:
-                self.format = args.pop("format")
-        if args:
-            raise AnsibleError(
-                "unrecognized arguments to with_sequence: %r"
-                % args.keys()
-            )
-
-    def parse_simple_args(self, term):
-        """parse the shortcut forms, return True/False"""
-        match = SHORTCUT.match(term)
-        if not match:
-            return False
-
-        _, start, end, _, stride, _, format = match.groups()
-
-        if start is not None:
-            try:
-                start = int(start, 0)
-            except ValueError:
-                raise AnsibleError("can't parse start=%s as integer" % start)
-        if end is not None:
-            try:
-                end = int(end, 0)
-            except ValueError:
-                raise AnsibleError("can't parse end=%s as integer" % end)
-        if stride is not None:
-            try:
-                stride = int(stride, 0)
-            except ValueError:
-                raise AnsibleError("can't parse stride=%s as integer" % stride)
-
-        if start is not None:
-            self.start = start
-        if end is not None:
-            self.end = end
-        if stride is not None:
-            self.stride = stride
-        if format is not None:
-            self.format = format
-
-    def sanity_check(self):
-        if self.count is None and self.end is None:
-            raise AnsibleError(
-                "must specify count or end in with_sequence"
-            )
-        elif self.count is not None and self.end is not None:
-            raise AnsibleError(
-                "can't specify both count and end in with_sequence"
-            )
-        elif self.count is not None:
-            # convert count to end
-            self.end = self.start + self.count * self.stride - 1
-            del self.count
-        if self.end < self.start:
-            raise AnsibleError("can't count backwards")
-        if self.format.count('%') != 1:
-            raise AnsibleError("bad formatting string: %s" % self.format)
-
-    def generate_sequence(self):
-        numbers = xrange(self.start, self.end + 1, self.stride)
-
-        for i in numbers:
-            try:
-                formatted = self.format % i
-                yield formatted
-            except (ValueError, TypeError):
-                raise AnsibleError(
-                    "problem formatting %r with %r" % self.format
-                )
-
-    def run(self, terms, inject=None, **kwargs):
-        results = []
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-
-        if isinstance(terms, basestring):
-            terms = [ terms ]
-
-        for term in terms:
-            try:
-                self.reset()  # clear out things for this iteration
-
-                try:
-                    if not self.parse_simple_args(term):
-                        self.parse_kv_args(utils.parse_kv(term))
-                except Exception:
-                    raise AnsibleError(
-                        "unknown error parsing with_sequence arguments: %r"
-                        % term
-                    )
-
-                self.sanity_check()
-
-                results.extend(self.generate_sequence())
-            except AnsibleError:
-                raise
-            except Exception:
-                raise AnsibleError(
-                    "unknown error generating sequence"
-                )
-
-        return results

v2/ansible/plugins/lookup/subelements.py

@@ -1,67 +0,0 @@
-# (c) 2013, Serge van Ginderachter <serge@vanginderachter.be>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-import ansible.utils as utils
-import ansible.errors as errors
-
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-
-    def run(self, terms, inject=None, **kwargs):
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-        terms[0] = utils.listify_lookup_plugin_terms(terms[0], self.basedir, inject)
-
-        if not isinstance(terms, list) or not len(terms) == 2:
-            raise errors.AnsibleError(
-                "subelements lookup expects a list of two items, first a dict or a list, and second a string")
-        terms[0] = utils.listify_lookup_plugin_terms(terms[0], self.basedir, inject)
-        if not isinstance(terms[0], (list, dict)) or not isinstance(terms[1], basestring):
-            raise errors.AnsibleError(
-                "subelements lookup expects a list of two items, first a dict or a list, and second a string")
-
-        if isinstance(terms[0], dict): # convert to list:
-            if terms[0].get('skipped',False) != False:
-                # the registered result was completely skipped
-                return []
-            elementlist = []
-            for key in terms[0].iterkeys():
-                elementlist.append(terms[0][key])
-        else: 
-            elementlist = terms[0]
-        subelement = terms[1]
-
-        ret = []
-        for item0 in elementlist:
-            if not isinstance(item0, dict):
-                raise errors.AnsibleError("subelements lookup expects a dictionary, got '%s'" %item0)
-            if item0.get('skipped',False) != False:
-                # this particular item is to be skipped
-                continue 
-            if not subelement in item0:
-                raise errors.AnsibleError("could not find '%s' key in iterated item '%s'" % (subelement, item0))
-            if not isinstance(item0[subelement], list):
-                raise errors.AnsibleError("the key %s should point to a list, got '%s'" % (subelement, item0[subelement]))
-            sublist = item0.pop(subelement, [])
-            for item1 in sublist:
-                ret.append((item0, item1))
-
-        return ret
-

v2/ansible/plugins/lookup/together.py

@@ -1,64 +0,0 @@
-# (c) 2013, Bradley Young <young.bradley@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-import ansible.utils as utils
-from ansible.utils import safe_eval
-import ansible.errors as errors
-from itertools import izip_longest
-
-def flatten(terms):
-    ret = []
-    for term in terms:
-        if isinstance(term, list):
-            ret.extend(term)
-        elif isinstance(term, tuple):
-            ret.extend(term)
-        else:
-            ret.append(term)
-    return ret
-
-class LookupModule(object):
-    """
-    Transpose a list of arrays:
-    [1, 2, 3], [4, 5, 6] -> [1, 4], [2, 5], [3, 6]
-    Replace any empty spots in 2nd array with None:
-    [1, 2], [3] -> [1, 3], [2, None]
-    """
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def __lookup_injects(self, terms, inject):
-        results = []
-        for x in terms:
-            intermediate = utils.listify_lookup_plugin_terms(x, self.basedir, inject)
-            results.append(intermediate)
-        return results
-
-    def run(self, terms, inject=None, **kwargs):
-
-        # this code is common with 'items.py' consider moving to utils if we need it again
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-        terms = self.__lookup_injects(terms, inject)
-
-        my_list = terms[:]
-        if len(my_list) == 0:
-            raise errors.AnsibleError("with_together requires at least one element in each list")
-        return [flatten(x) for x in izip_longest(*my_list, fillvalue=None)]
-
-

v2/ansible/plugins/shell/csh.py

@@ -1,6 +1,6 @@
-# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2014, Chris Church <chris@ninemoreminutes.com>
 #
-# This file is part of Ansible
+# This file is part of Ansible.
 #
 # Ansible is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,25 +15,9 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
-import glob
-from ansible import utils
+from ansible.runner.shell_plugins.sh import ShellModule as ShModule
 
-class LookupModule(object):
+class ShellModule(ShModule):
 
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def run(self, terms, inject=None, **kwargs):
-
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject)
-
-        ret = []
-
-        for term in terms:
-
-            dwimmed = utils.path_dwim(self.basedir, term)
-            globbed = glob.glob(dwimmed)
-            ret.extend(g for g in globbed if os.path.isfile(g))
-
-        return ret
+    def env_prefix(self, **kwargs):
+        return 'env %s' % super(ShellModule, self).env_prefix(**kwargs)

v2/ansible/plugins/shell/fish.py

@@ -0,0 +1,23 @@
+# (c) 2014, Chris Church <chris@ninemoreminutes.com>
+#
+# This file is part of Ansible.
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from ansible.runner.shell_plugins.sh import ShellModule as ShModule
+
+class ShellModule(ShModule):
+
+    def env_prefix(self, **kwargs):
+        return 'env %s' % super(ShellModule, self).env_prefix(**kwargs)

v2/ansible/plugins/shell/powershell.py

@@ -0,0 +1,117 @@
+# (c) 2014, Chris Church <chris@ninemoreminutes.com>
+#
+# This file is part of Ansible.
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+import os
+import re
+import random
+import shlex
+import time
+
+_common_args = ['PowerShell', '-NoProfile', '-NonInteractive']
+
+# Primarily for testing, allow explicitly specifying PowerShell version via
+# an environment variable.
+_powershell_version = os.environ.get('POWERSHELL_VERSION', None)
+if _powershell_version:
+    _common_args = ['PowerShell', '-Version', _powershell_version] + _common_args[1:]
+
+def _escape(value, include_vars=False):
+    '''Return value escaped for use in PowerShell command.'''
+    # http://www.techotopia.com/index.php/Windows_PowerShell_1.0_String_Quoting_and_Escape_Sequences
+    # http://stackoverflow.com/questions/764360/a-list-of-string-replacements-in-python
+    subs = [('\n', '`n'), ('\r', '`r'), ('\t', '`t'), ('\a', '`a'),
+            ('\b', '`b'), ('\f', '`f'), ('\v', '`v'), ('"', '`"'),
+            ('\'', '`\''), ('`', '``'), ('\x00', '`0')]
+    if include_vars:
+        subs.append(('$', '`$'))
+    pattern = '|'.join('(%s)' % re.escape(p) for p, s in subs)
+    substs = [s for p, s in subs]
+    replace = lambda m: substs[m.lastindex - 1]
+    return re.sub(pattern, replace, value)
+
+def _encode_script(script, as_list=False):
+    '''Convert a PowerShell script to a single base64-encoded command.'''
+    script = '\n'.join([x.strip() for x in script.splitlines() if x.strip()])
+    encoded_script = base64.b64encode(script.encode('utf-16-le'))
+    cmd_parts = _common_args + ['-EncodedCommand', encoded_script]
+    if as_list:
+        return cmd_parts
+    return ' '.join(cmd_parts)
+
+def _build_file_cmd(cmd_parts):
+    '''Build command line to run a file, given list of file name plus args.'''
+    return ' '.join(_common_args + ['-ExecutionPolicy', 'Unrestricted', '-File'] + ['"%s"' % x for x in cmd_parts])
+
+class ShellModule(object):
+
+    def env_prefix(self, **kwargs):
+        return ''
+
+    def join_path(self, *args):
+        return os.path.join(*args).replace('/', '\\')
+
+    def path_has_trailing_slash(self, path):
+        # Allow Windows paths to be specified using either slash.
+        return path.endswith('/') or path.endswith('\\')
+
+    def chmod(self, mode, path):
+        return ''
+
+    def remove(self, path, recurse=False):
+        path = _escape(path)
+        if recurse:
+            return _encode_script('''Remove-Item "%s" -Force -Recurse;''' % path)
+        else:
+            return _encode_script('''Remove-Item "%s" -Force;''' % path)
+
+    def mkdtemp(self, basefile, system=False, mode=None):
+        basefile = _escape(basefile)
+        # FIXME: Support system temp path!
+        return _encode_script('''(New-Item -Type Directory -Path $env:temp -Name "%s").FullName | Write-Host -Separator '';''' % basefile)
+
+    def md5(self, path):
+        path = _escape(path)
+        script = '''
+            If (Test-Path -PathType Leaf "%(path)s")
+            {
+                $sp = new-object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider;
+                $fp = [System.IO.File]::Open("%(path)s", [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read);
+                [System.BitConverter]::ToString($sp.ComputeHash($fp)).Replace("-", "").ToLower();
+                $fp.Dispose();
+            }
+            ElseIf (Test-Path -PathType Container "%(path)s")
+            {
+                Write-Host "3";
+            }
+            Else
+            {
+                Write-Host "1";
+            }
+        ''' % dict(path=path)
+        return _encode_script(script)
+
+    def build_module_command(self, env_string, shebang, cmd, rm_tmp=None):
+        cmd = cmd.encode('utf-8')
+        cmd_parts = shlex.split(cmd, posix=False)
+        if not cmd_parts[0].lower().endswith('.ps1'):
+            cmd_parts[0] = '%s.ps1' % cmd_parts[0]
+        script = _build_file_cmd(cmd_parts)
+        if rm_tmp:
+            rm_tmp = _escape(rm_tmp)
+            script = '%s; Remove-Item "%s" -Force -Recurse;' % (script, rm_tmp)
+        return _encode_script(script)

v2/ansible/plugins/shell/sh.py

@@ -0,0 +1,115 @@
+# (c) 2014, Chris Church <chris@ninemoreminutes.com>
+#
+# This file is part of Ansible.
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+import pipes
+import ansible.constants as C
+
+_USER_HOME_PATH_RE = re.compile(r'^~[_.A-Za-z0-9][-_.A-Za-z0-9]*$')
+
+class ShellModule(object):
+
+    def env_prefix(self, **kwargs):
+        '''Build command prefix with environment variables.'''
+        env = dict(
+            LANG     = C.DEFAULT_MODULE_LANG,
+            LC_CTYPE = C.DEFAULT_MODULE_LANG,
+        )
+        env.update(kwargs)
+        return ' '.join(['%s=%s' % (k, pipes.quote(unicode(v))) for k,v in env.items()])
+
+    def join_path(self, *args):
+        return os.path.join(*args)
+
+    def path_has_trailing_slash(self, path):
+        return path.endswith('/')
+
+    def chmod(self, mode, path):
+        path = pipes.quote(path)
+        return 'chmod %s %s' % (mode, path)
+
+    def remove(self, path, recurse=False):
+        path = pipes.quote(path)
+        if recurse:
+            return "rm -rf %s >/dev/null 2>&1" % path
+        else:
+            return "rm -f %s >/dev/null 2>&1" % path
+
+    def mkdtemp(self, basefile=None, system=False, mode=None):
+        if not basefile:
+            basefile = 'ansible-tmp-%s-%s' % (time.time(), random.randint(0, 2**48))
+        basetmp = self.join_path(C.DEFAULT_REMOTE_TMP, basefile)
+        if system and basetmp.startswith('$HOME'):
+            basetmp = self.join_path('/tmp', basefile)
+        cmd = 'mkdir -p %s' % basetmp
+        if mode:
+            cmd += ' && chmod %s %s' % (mode, basetmp)
+        cmd += ' && echo %s' % basetmp
+        return cmd
+
+    def expand_user(self, user_home_path):
+        ''' Return a command to expand tildes in a path
+
+        It can be either "~" or "~username".  We use the POSIX definition of
+        a username:
+            http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_426
+            http://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_276
+        '''
+
+        # Check that the user_path to expand is safe
+        if user_home_path != '~':
+            if not _USER_HOME_PATH_RE.match(user_home_path):
+                # pipes.quote will make the shell return the string verbatim
+                user_home_path = pipes.quote(user_home_path)
+        return 'echo %s' % user_home_path
+
+    def checksum(self, path, python_interp):
+        path = pipes.quote(path)
+        # The following test needs to be SH-compliant.  BASH-isms will
+        # not work if /bin/sh points to a non-BASH shell.
+        #
+        # In the following test, each condition is a check and logical
+        # comparison (|| or &&) that sets the rc value.  Every check is run so
+        # the last check in the series to fail will be the rc that is
+        # returned.
+        #
+        # If a check fails we error before invoking the hash functions because
+        # hash functions may successfully take the hash of a directory on BSDs
+        # (UFS filesystem?) which is not what the rest of the ansible code
+        # expects
+        #
+        # If all of the available hashing methods fail we fail with an rc of
+        # 0.  This logic is added to the end of the cmd at the bottom of this
+        # function.
+
+        test = "rc=flag; [ -r \"%(p)s\" ] || rc=2; [ -f \"%(p)s\" ] || rc=1; [ -d \"%(p)s\" ] && rc=3; %(i)s -V 2>/dev/null || rc=4; [ x\"$rc\" != \"xflag\" ] && echo \"${rc}  %(p)s\" && exit 0" % dict(p=path, i=python_interp)
+        csums = [
+            "(%s -c 'import hashlib; print(hashlib.sha1(open(\"%s\", \"rb\").read()).hexdigest())' 2>/dev/null)" % (python_interp, path),      # Python > 2.4 (including python3)
+            "(%s -c 'import sha; print(sha.sha(open(\"%s\", \"rb\").read()).hexdigest())' 2>/dev/null)" % (python_interp, path),        # Python == 2.4
+        ]
+
+        cmd = " || ".join(csums)
+        cmd = "%s; %s || (echo \"0  %s\")" % (test, cmd, path)
+        return cmd
+
+    def build_module_command(self, env_string, shebang, cmd, rm_tmp=None):
+        cmd_parts = [env_string.strip(), shebang.replace("#!", "").strip(), cmd]
+        new_cmd = " ".join(cmd_parts)
+        if rm_tmp:
+            new_cmd = '%s; rm -rf %s >/dev/null 2>&1' % (new_cmd, rm_tmp)
+        return new_cmd

v2/ansible/plugins/strategies/__init__.py

@@ -0,0 +1,282 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import Queue
+import time
+
+from ansible.errors import *
+from ansible.playbook.helpers import compile_block_list
+from ansible.playbook.role import ROLE_CACHE
+from ansible.utils.debug import debug
+
+
+__all__ = ['StrategyBase']
+
+
+class StrategyBase:
+
+    '''
+    This is the base class for strategy plugins, which contains some common
+    code useful to all strategies like running handlers, cleanup actions, etc.
+    '''
+
+    def __init__(self, tqm):
+        self._tqm               = tqm
+        self._inventory         = tqm.get_inventory()
+        self._workers           = tqm.get_workers()
+        self._notified_handlers = tqm.get_notified_handlers()
+        self._callback          = tqm.get_callback()
+        self._variable_manager  = tqm.get_variable_manager()
+        self._loader            = tqm.get_loader()
+        self._final_q           = tqm._final_q
+
+        # internal counters
+        self._pending_results   = 0
+        self._cur_worker        = 0
+
+        # this dictionary is used to keep track of hosts that have
+        # outstanding tasks still in queue
+        self._blocked_hosts     = dict()
+
+    def run(self, iterator, connection_info):
+        debug("running the cleanup portion of the play")
+        result = self.cleanup(iterator, connection_info)
+        debug("running handlers")
+        result &= self.run_handlers(iterator, connection_info)
+        return result
+
+    def get_hosts_remaining(self, play):
+        return [host for host in self._inventory.get_hosts(play.hosts) if host.name not in self._tqm._failed_hosts and host.get_name() not in self._tqm._unreachable_hosts]
+
+    def get_failed_hosts(self):
+        return [host for host in self._inventory.get_hosts() if host.name in self._tqm._failed_hosts]
+
+    def _queue_task(self, play, host, task, connection_info):
+        ''' handles queueing the task up to be sent to a worker '''
+
+        debug("entering _queue_task() for %s/%s/%s" % (play, host, task))
+        # copy the task, to make sure we have a clean version, since the
+        # post-validation step will alter attribute values but this Task object
+        # is shared across all hosts in the play
+        debug("copying task")
+        new_task = task.copy()
+        debug("done copying task")
+
+        # squash variables down to a single dictionary using the variable manager and
+        # call post_validate() on the task, which will finalize the attribute values
+        debug("getting variables")
+        try:
+            task_vars = self._variable_manager.get_vars(loader=self._loader, play=play, host=host, task=new_task)
+        except EOFError:
+            # usually happens if the program is aborted, and the proxied object
+            # queue is cut off from the call, so we just ignore this and exit
+            return
+        debug("done getting variables")
+
+        debug("running post_validate() on the task")
+        new_task.post_validate(task_vars)
+        debug("done running post_validate() on the task")
+
+        # and then queue the new task
+        debug("%s - putting task (%s) in queue" % (host, task))
+        try:
+            debug("worker is %d (out of %d available)" % (self._cur_worker+1, len(self._workers)))
+
+            (worker_prc, main_q, rslt_q) = self._workers[self._cur_worker]
+            self._cur_worker += 1
+            if self._cur_worker >= len(self._workers):
+                self._cur_worker = 0
+
+            self._pending_results += 1
+            main_q.put((host, new_task, task_vars, connection_info), block=False)
+        except (EOFError, IOError, AssertionError), e:
+            # most likely an abort
+            debug("got an error while queuing: %s" % e)
+            return
+        debug("exiting _queue_task() for %s/%s/%s" % (play, host, task))
+
+    def _process_pending_results(self):
+        '''
+        Reads results off the final queue and takes appropriate action
+        based on the result (executing callbacks, updating state, etc.).
+        '''
+
+        while not self._final_q.empty() and not self._tqm._terminated:
+            try:
+                result = self._final_q.get(block=False)
+                debug("got result from result worker: %s" % (result,))
+
+                # all host status messages contain 2 entries: (msg, task_result)
+                if result[0] in ('host_task_ok', 'host_task_failed', 'host_task_skipped', 'host_unreachable'):
+                    task_result = result[1]
+                    host = task_result._host
+                    task = task_result._task
+                    if result[0] == 'host_task_failed':
+                        self._tqm._failed_hosts[host.get_name()] = True
+                        self._callback.runner_on_failed(task, task_result)
+                    elif result[0] == 'host_unreachable':
+                        self._tqm._unreachable_hosts[host.get_name()] = True
+                        self._callback.runner_on_unreachable(task, task_result)
+                    elif result[0] == 'host_task_skipped':
+                        self._callback.runner_on_skipped(task, task_result)
+                    elif result[0] == 'host_task_ok':
+                        self._callback.runner_on_ok(task, task_result)
+
+                    self._pending_results -= 1
+                    if host.name in self._blocked_hosts:
+                        del self._blocked_hosts[host.name]
+
+                    # If this is a role task, mark the parent role as being run (if
+                    # the task was ok or failed, but not skipped or unreachable)
+                    if task_result._task._role is not None and result[0] in ('host_task_ok', 'host_task_failed'):
+                        # lookup the role in the ROLE_CACHE to make sure we're dealing
+                        # with the correct object and mark it as executed
+                        for (entry, role_obj) in ROLE_CACHE[task_result._task._role._role_name].iteritems():
+                            hashed_entry = frozenset(task_result._task._role._role_params.iteritems())
+                            if entry == hashed_entry :
+                                role_obj._had_task_run = True
+
+                elif result[0] == 'notify_handler':
+                    handler_name = result[1]
+                    host         = result[2]
+                    if host not in self._notified_handlers[handler_name]:
+                        self._notified_handlers[handler_name].append(host)
+
+                elif result[0] == 'set_host_var':
+                    host      = result[1]
+                    var_name  = result[2]
+                    var_value = result[3]
+                    self._variable_manager.set_host_variable(host, var_name, var_value)
+
+                elif result[0] == 'set_host_facts':
+                    host  = result[1]
+                    facts = result[2]
+                    self._variable_manager.set_host_facts(host, facts)
+
+                else:
+                    raise AnsibleError("unknown result message received: %s" % result[0])
+            except Queue.Empty:
+                pass
+
+    def _wait_on_pending_results(self):
+        '''
+        Wait for the shared counter to drop to zero, using a short sleep
+        between checks to ensure we don't spin lock
+        '''
+
+        while self._pending_results > 0 and not self._tqm._terminated:
+            debug("waiting for pending results (%d left)" % self._pending_results)
+            self._process_pending_results()
+            if self._tqm._terminated:
+                break
+            time.sleep(0.01)
+
+    def cleanup(self, iterator, connection_info):
+        '''
+        Iterates through failed hosts and runs any outstanding rescue/always blocks
+        and handlers which may still need to be run after a failure.
+        '''
+
+        debug("in cleanup")
+        result = True
+
+        debug("getting failed hosts")
+        failed_hosts = self.get_failed_hosts()
+        if len(failed_hosts) == 0:
+            debug("there are no failed hosts")
+            return result
+
+        debug("marking hosts failed in the iterator")
+        # mark the host as failed in the iterator so it will take
+        # any required rescue paths which may be outstanding
+        for host in failed_hosts:
+            iterator.mark_host_failed(host)
+
+        debug("clearing the failed hosts list")
+        # clear the failed hosts dictionary now while also
+        for entry in self._tqm._failed_hosts.keys():
+            del self._tqm._failed_hosts[entry]
+
+        work_to_do = True
+        while work_to_do:
+            work_to_do = False
+            for host in failed_hosts:
+                host_name = host.get_name()
+
+                if host_name in self._tqm._failed_hosts:
+                    iterator.mark_host_failed(host)
+                    del self._tqm._failed_hosts[host_name]
+
+                if host_name not in self._tqm._unreachable_hosts and iterator.get_next_task_for_host(host, peek=True):
+                    work_to_do = True
+                    # check to see if this host is blocked (still executing a previous task)
+                    if not host_name in self._blocked_hosts:
+                        # pop the task, mark the host blocked, and queue it
+                        self._blocked_hosts[host_name] = True
+                        task = iterator.get_next_task_for_host(host)
+                        self._callback.playbook_on_cleanup_task_start(task.get_name())
+                        self._queue_task(iterator._play, host, task, connection_info)
+
+            self._process_pending_results()
+
+        # no more work, wait until the queue is drained
+        self._wait_on_pending_results()
+
+        return result
+
+    def run_handlers(self, iterator, connection_info):
+        '''
+        Runs handlers on those hosts which have been notified.
+        '''
+
+        result = True
+
+        # FIXME: getting the handlers from the iterators play should be
+        #        a method on the iterator, which may also filter the list
+        #        of handlers based on the notified list
+        handlers = compile_block_list(iterator._play.handlers)
+
+        debug("handlers are: %s" % handlers)
+        for handler in handlers:
+            handler_name = handler.get_name()
+
+            if handler_name in self._notified_handlers and len(self._notified_handlers[handler_name]):
+                if not len(self.get_hosts_remaining()):
+                    self._callback.playbook_on_no_hosts_remaining()
+                    result = False
+                    break
+
+                self._callback.playbook_on_handler_task_start(handler_name)
+                for host in self._notified_handlers[handler_name]:
+                    if not handler.has_triggered(host):
+                        temp_data = handler.serialize()
+                        self._queue_task(iterator._play, host, handler, connection_info)
+                        handler.flag_for_host(host)
+
+                    self._process_pending_results()
+
+                self._wait_on_pending_results()
+
+                # wipe the notification list
+                self._notified_handlers[handler_name] = []
+
+        debug("done running handlers, result is: %s" % result)
+        return result

v2/ansible/plugins/strategies/free.py

@@ -0,0 +1,105 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import time
+
+from ansible.plugins.strategies import StrategyBase
+
+class StrategyModule(StrategyBase):
+
+    def run(self, iterator, connection_info):
+        '''
+        The "free" strategy is a bit more complex, in that it allows tasks to
+        be sent to hosts as quickly as they can be processed. This means that
+        some hosts may finish very quickly if run tasks result in little or no
+        work being done versus other systems.
+
+        The algorithm used here also tries to be more "fair" when iterating
+        through hosts by remembering the last host in the list to be given a task
+        and starting the search from there as opposed to the top of the hosts
+        list again, which would end up favoring hosts near the beginning of the
+        list.
+        '''
+
+        # the last host to be given a task
+        last_host = 0      
+
+        work_to_do = True
+        while work_to_do:
+
+            hosts_left = self.get_hosts_remaining()
+            if len(hosts_left) == 0:
+                self._callback.playbook_on_no_hosts_remaining()
+                break
+
+            # using .qsize() is a best estimate anyway, due to the
+            # multiprocessing/threading concerns (per the python docs)
+            if 1: #if self._job_queue.qsize() < len(hosts_left):
+
+                work_to_do = False        # assume we have no more work to do
+                starting_host = last_host # save current position so we know when we've
+                                          # looped back around and need to break
+
+                # try and find an unblocked host with a task to run
+                while True:
+                    host = hosts_left[last_host]
+                    host_name = host.get_name()
+
+                    # peek at the next task for the host, to see if there's
+                    # anything to do do for this host
+                    if host_name not in self._tqm._failed_hosts and host_name not in self._tqm._unreachable_hosts and iterator.get_next_task_for_host(host, peek=True):
+
+                        # set the flag so the outer loop knows we've still found
+                        # some work which needs to be done
+                        work_to_do = True
+
+                        # check to see if this host is blocked (still executing a previous task)
+                        if not host_name in self._blocked_hosts:
+                            # pop the task, mark the host blocked, and queue it
+                            self._blocked_hosts[host_name] = True
+                            task = iterator.get_next_task_for_host(host)
+                            #self._callback.playbook_on_task_start(task.get_name(), False)
+                            self._queue_task(iterator._play, host, task, connection_info)
+
+                    # move on to the next host and make sure we
+                    # haven't gone past the end of our hosts list
+                    last_host += 1
+                    if last_host > len(hosts_left) - 1:
+                        last_host = 0
+
+                    # if we've looped around back to the start, break out
+                    if last_host == starting_host:
+                        break
+
+            # pause briefly so we don't spin lock
+            time.sleep(0.05)
+
+        try:
+            self._wait_for_pending_results()
+        except:
+            # FIXME: ctrl+c can cause some failures here, so catch them
+            #        with the appropriate error type
+            pass
+
+        # run the base class run() method, which executes the cleanup function
+        # and runs any outstanding handlers which have been triggered
+        super(StrategyModule, self).run(iterator, connection_info)
+

v2/ansible/plugins/strategies/linear.py

@@ -0,0 +1,84 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.plugins.strategies import StrategyBase
+
+from ansible.utils.debug import debug
+
+class StrategyModule(StrategyBase):
+
+    def run(self, iterator, connection_info):
+        '''
+        The linear strategy is simple - get the next task and queue
+        it for all hosts, then wait for the queue to drain before
+        moving on to the next task
+        '''
+
+        result = True
+
+        # iteratate over each task, while there is one left to run
+        work_to_do = True
+        while work_to_do:
+
+            try:
+                debug("getting the remaining hosts for this loop")
+                hosts_left = self.get_hosts_remaining(iterator._play)
+                debug("done getting the remaining hosts for this loop")
+                if len(hosts_left) == 0:
+                    debug("out of hosts to run on")
+                    self._callback.playbook_on_no_hosts_remaining()
+                    result = False
+                    break
+
+                # queue up this task for each host in the inventory
+                callback_sent = False
+                work_to_do = False
+                for host in hosts_left:
+                    task = iterator.get_next_task_for_host(host)
+                    if not task:
+                        continue
+
+                    work_to_do = True
+                    if not callback_sent:
+                        self._callback.playbook_on_task_start(task.get_name(), False)
+                        callback_sent = True
+
+                    host_name = host.get_name()
+                    if 1: #host_name not in self._tqm._failed_hosts and host_name not in self._tqm._unreachable_hosts:
+                        self._blocked_hosts[host_name] = True
+                        self._queue_task(iterator._play, host, task, connection_info)
+
+                    self._process_pending_results()
+
+                debug("done queuing things up, now waiting for results queue to drain")
+                self._wait_on_pending_results()
+                debug("results queue empty")
+            except IOError, e:
+                debug("got IOError: %s" % e)
+                # most likely an abort, return failed
+                return 1
+
+        # run the base class run() method, which executes the cleanup function
+        # and runs any outstanding handlers which have been triggered
+
+        result &= super(StrategyModule, self).run(iterator, connection_info)
+
+        return result