Add timeout support to ansible-test. (#53302)

* Add timeout support to ansible-test.
* Fix ansible-test tar filename filter bug.
* Update timeouts used on Shippable.
* Kill subprocesses when parent process terminates.
* Require explicit use of env --show option.

test/runner/lib/env.py

@@ -4,13 +4,17 @@ from __future__ import absolute_import, print_function
 
 import datetime
 import json
+import functools
 import os
 import platform
 import re
+import signal
 import sys
+import time
 
 from lib.config import (
     CommonConfig,
+    TestConfig,
 )
 
 from lib.util import (
@@ -34,6 +38,18 @@ from lib.docker_util import (
     docker_version
 )
 
+from lib.thread import (
+    WrappedThread,
+)
+
+from lib.constants import (
+    TIMEOUT_PATH,
+)
+
+from lib.test import (
+    TestTimeout,
+)
+
 
 class EnvConfig(CommonConfig):
     """Configuration for the tools command."""
@@ -43,14 +59,26 @@ class EnvConfig(CommonConfig):
         """
         super(EnvConfig, self).__init__(args, 'env')
 
-        self.show = args.show or not args.dump
+        self.show = args.show
         self.dump = args.dump
+        self.timeout = args.timeout
 
 
 def command_env(args):
     """
     :type args: EnvConfig
     """
+    show_dump_env(args)
+    set_timeout(args)
+
+
+def show_dump_env(args):
+    """
+    :type args: EnvConfig
+    """
+    if not args.show and not args.dump:
+        return
+
     data = dict(
         ansible=dict(
             version=get_ansible_version(args),
@@ -84,6 +112,105 @@ def command_env(args):
             results_fd.write(json.dumps(data, sort_keys=True))
 
 
+def set_timeout(args):
+    """
+    :type args: EnvConfig
+    """
+    if args.timeout is None:
+        return
+
+    if args.timeout:
+        deadline = (datetime.datetime.utcnow() + datetime.timedelta(minutes=args.timeout)).strftime('%Y-%m-%dT%H:%M:%SZ')
+
+        display.info('Setting a %d minute test timeout which will end at: %s' % (args.timeout, deadline), verbosity=1)
+    else:
+        deadline = None
+
+        display.info('Clearing existing test timeout.', verbosity=1)
+
+    if args.explain:
+        return
+
+    if deadline:
+        data = dict(
+            duration=args.timeout,
+            deadline=deadline,
+        )
+
+        with open(TIMEOUT_PATH, 'w') as timeout_fd:
+            json.dump(data, timeout_fd, indent=4, sort_keys=True)
+    elif os.path.exists(TIMEOUT_PATH):
+        os.remove(TIMEOUT_PATH)
+
+
+def get_timeout():
+    """
+    :rtype: dict[str, any] | None
+    """
+    if not os.path.exists(TIMEOUT_PATH):
+        return None
+
+    with open(TIMEOUT_PATH, 'r') as timeout_fd:
+        data = json.load(timeout_fd)
+
+    data['deadline'] = datetime.datetime.strptime(data['deadline'], '%Y-%m-%dT%H:%M:%SZ')
+
+    return data
+
+
+def configure_timeout(args):
+    """
+    :type args: CommonConfig
+    """
+    if isinstance(args, TestConfig):
+        configure_test_timeout(args)  # only tests are subject to the timeout
+
+
+def configure_test_timeout(args):
+    """
+    :type args: TestConfig
+    """
+    timeout = get_timeout()
+
+    if not timeout:
+        return
+
+    timeout_start = datetime.datetime.utcnow()
+    timeout_duration = timeout['duration']
+    timeout_deadline = timeout['deadline']
+    timeout_remaining = timeout_deadline - timeout_start
+
+    test_timeout = TestTimeout(timeout_duration)
+
+    if timeout_remaining <= datetime.timedelta():
+        test_timeout.write(args)
+
+        raise ApplicationError('The %d minute test timeout expired %s ago at %s.' % (
+            timeout_duration, timeout_remaining * -1, timeout_deadline))
+
+    display.info('The %d minute test timeout expires in %s at %s.' % (
+        timeout_duration, timeout_remaining, timeout_deadline), verbosity=1)
+
+    def timeout_handler(_dummy1, _dummy2):
+        """Runs when SIGUSR1 is received."""
+        test_timeout.write(args)
+
+        raise ApplicationError('Tests aborted after exceeding the %d minute time limit.' % timeout_duration)
+
+    def timeout_waiter(timeout_seconds):
+        """
+        :type timeout_seconds: int
+        """
+        time.sleep(timeout_seconds)
+        os.kill(os.getpid(), signal.SIGUSR1)
+
+    signal.signal(signal.SIGUSR1, timeout_handler)
+
+    instance = WrappedThread(functools.partial(timeout_waiter, timeout_remaining.seconds))
+    instance.daemon = True
+    instance.start()
+
+
 def show_dict(data, verbose, root_verbosity=0, path=None):
     """
     :type data: dict[str, any]