Test that transient failure in the middle of running async_status doesn't fail the task

test/integration/targets/async_fail/action_plugins/normal.py

@@ -0,0 +1,62 @@
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.errors import AnsibleError
+from ansible.plugins.action import ActionBase
+from ansible.utils.vars import merge_hash
+
+
+class ActionModule(ActionBase):
+
+    def run(self, tmp=None, task_vars=None):
+
+        # individual modules might disagree but as the generic the action plugin, pass at this point.
+        self._supports_check_mode = True
+        self._supports_async = True
+
+        result = super(ActionModule, self).run(tmp, task_vars)
+        del tmp  # tmp no longer has any effect
+
+        if not result.get('skipped'):
+
+            if result.get('invocation', {}).get('module_args'):
+                # avoid passing to modules in case of no_log
+                # should not be set anymore but here for backwards compatibility
+                del result['invocation']['module_args']
+
+            # FUTURE: better to let _execute_module calculate this internally?
+            wrap_async = self._task.async_val and not self._connection.has_native_async
+
+            # do work!
+            result = merge_hash(result, self._execute_module(task_vars=task_vars, wrap_async=wrap_async))
+
+            # hack to keep --verbose from showing all the setup module result
+            # moved from setup module as now we filter out all _ansible_ from result
+            if self._task.action == 'setup':
+                result['_ansible_verbose_override'] = True
+
+        # Simulate a transient network failure
+        if self._task.action == 'async_status' and 'finished' in result and result['finished'] != 1:
+            raise AnsibleError('Pretend to fail somewher ein executing async_status')
+
+        if not wrap_async:
+            # remove a temporary path we created
+            self._remove_tmp_path(self._connection._shell.tempdir)
+
+        return result

test/integration/targets/async_fail/aliases

@@ -0,0 +1,3 @@
+async_status
+async_wrapper
+posix/ci/group2

test/integration/targets/async_fail/library/async_test.py

@@ -0,0 +1,49 @@
+import json
+import sys
+import time
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main():
+    if "--interactive" in sys.argv:
+        import ansible.module_utils.basic
+        ansible.module_utils.basic._ANSIBLE_ARGS = json.dumps(dict(
+            ANSIBLE_MODULE_ARGS=dict(
+                fail_mode="graceful"
+            )
+        ))
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            fail_mode=dict(type='list', default=['success'])
+        )
+    )
+
+    result = dict(changed=True)
+
+    fail_mode = module.params['fail_mode']
+
+    try:
+        if 'leading_junk' in fail_mode:
+            print("leading junk before module output")
+
+        if 'graceful' in fail_mode:
+            module.fail_json(msg="failed gracefully")
+
+        if 'exception' in fail_mode:
+            raise Exception('failing via exception')
+
+        if 'recovered_fail' in fail_mode:
+            result = {"msg": "succeeded", "failed": False, "changed": True}
+            # Wait in the middle to setup a race where the controller reads incomplete data from our
+            # special async_status the first poll
+            time.sleep(5)
+
+        module.exit_json(**result)
+
+    finally:
+        if 'trailing_junk' in fail_mode:
+            print("trailing junk after module output")
+
+main()

test/integration/targets/async_fail/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - prepare_tests

test/integration/targets/async_fail/tasks/main.yml

@@ -0,0 +1,35 @@
+# test code for the async keyword failing in the middle of output
+# (c) 2018, Ansible Project
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# This uses a special copy of the normal action plugin which simulates
+# a transient failure in the module
+- name: test that we can recover from initial failures to read
+  async_test:
+    fail_mode: recovered_fail
+  async: 10
+  poll: 1
+  register: async_result
+
+- name: validate that by the end of the retry interval, we succeeded
+  assert:
+    that:
+    - async_result.ansible_job_id is match('\d+\.\d+')
+    - async_result.finished == 1
+    - async_result.changed == true
+    - async_result is successful
+    - async_result.msg is search('succeeded')