diff --git a/test/runner/lib/ansible_util.py b/test/runner/lib/ansible_util.py index 512be0c024..c17abcf393 100644 --- a/test/runner/lib/ansible_util.py +++ b/test/runner/lib/ansible_util.py @@ -4,6 +4,10 @@ from __future__ import absolute_import, print_function import os +from lib.constants import ( + SOFT_RLIMIT_NOFILE, +) + from lib.util import ( common_environment, ApplicationError, @@ -40,7 +44,7 @@ def ansible_environment(args, color=True, ansible_config=None): raise ApplicationError('Configuration not found: %s' % ansible_config) ansible = dict( - ANSIBLE_PYTHON_MODULE_RLIMIT_NOFILE='1024', + ANSIBLE_PYTHON_MODULE_RLIMIT_NOFILE=str(SOFT_RLIMIT_NOFILE), ANSIBLE_FORCE_COLOR='%s' % 'true' if args.color and color else 'false', ANSIBLE_DEPRECATION_WARNINGS='false', ANSIBLE_HOST_KEY_CHECKING='false', diff --git a/test/runner/lib/cli.py b/test/runner/lib/cli.py index 86ecbe4a36..933a16e1b5 100644 --- a/test/runner/lib/cli.py +++ b/test/runner/lib/cli.py @@ -4,9 +4,14 @@ from __future__ import absolute_import, print_function import errno import os -import resource import sys +# This import should occur as early as possible. +# It must occur before subprocess has been imported anywhere in the current process. +from lib.init import ( + CURRENT_RLIMIT_NOFILE, +) + from lib.util import ( ApplicationError, display, @@ -14,6 +19,7 @@ from lib.util import ( get_docker_completion, generate_pip_command, read_lines_without_comments, + MAXFD, ) from lib.delegation import ( @@ -90,16 +96,8 @@ def main(): display.info_stderr = (isinstance(config, SanityConfig) and config.lint) or (isinstance(config, IntegrationConfig) and config.list_targets) check_startup() - # to achieve a consistent nofile ulimit, set to 16k here, this can affect performance in subprocess.Popen when - # being called with close_fds=True on Python (8x the time on some environments) - nofile_limit = 16 * 1024 - current_limit = resource.getrlimit(resource.RLIMIT_NOFILE) - new_limit = (nofile_limit, nofile_limit) - if current_limit > new_limit: - display.info('RLIMIT_NOFILE: %s -> %s' % (current_limit, new_limit), verbosity=2) - resource.setrlimit(resource.RLIMIT_NOFILE, (nofile_limit, nofile_limit)) - else: - display.info('RLIMIT_NOFILE: %s' % (current_limit, ), verbosity=2) + display.info('RLIMIT_NOFILE: %s' % (CURRENT_RLIMIT_NOFILE,), verbosity=2) + display.info('MAXFD: %d' % MAXFD, verbosity=2) try: args.func(config) diff --git a/test/runner/lib/constants.py b/test/runner/lib/constants.py new file mode 100644 index 0000000000..25096742fd --- /dev/null +++ b/test/runner/lib/constants.py @@ -0,0 +1,5 @@ +"""Constants used by ansible-test. Imports should not be used in this file.""" + +# Setting a low soft RLIMIT_NOFILE value will improve the performance of subprocess.Popen on Python 2.x when close_fds=True. +# This will affect all Python subprocesses. It will also affect the current Python process if set before subprocess is imported for the first time. +SOFT_RLIMIT_NOFILE = 1024 diff --git a/test/runner/lib/init.py b/test/runner/lib/init.py new file mode 100644 index 0000000000..7bc39c3ed0 --- /dev/null +++ b/test/runner/lib/init.py @@ -0,0 +1,14 @@ +"""Early initialization for ansible-test before most other imports have been performed.""" + +import resource + +from lib.constants import ( + SOFT_RLIMIT_NOFILE, +) + +CURRENT_RLIMIT_NOFILE = resource.getrlimit(resource.RLIMIT_NOFILE) +DESIRED_RLIMIT_NOFILE = (SOFT_RLIMIT_NOFILE, CURRENT_RLIMIT_NOFILE[1]) + +if DESIRED_RLIMIT_NOFILE < CURRENT_RLIMIT_NOFILE: + resource.setrlimit(resource.RLIMIT_NOFILE, DESIRED_RLIMIT_NOFILE) + CURRENT_RLIMIT_NOFILE = DESIRED_RLIMIT_NOFILE diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py index 47e51849fd..9838b0eab6 100644 --- a/test/runner/lib/util.py +++ b/test/runner/lib/util.py @@ -41,6 +41,11 @@ except ImportError: DOCKER_COMPLETION = {} COVERAGE_PATHS = {} # type: dict[str, str] +try: + MAXFD = subprocess.MAXFD +except AttributeError: + MAXFD = -1 + def get_docker_completion(): """