From 87a9485b2f5a3188460f0a0219d2e0d990ce4e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannig=20Perr=C3=A9?= Date: Sun, 1 Nov 2015 22:30:55 +0100 Subject: [PATCH 1/7] Cache for _do_template call. May result in nice speed improvement (4-5 times faster). --- lib/ansible/template/__init__.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index b7a0ea7c87..165b9e0e96 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -122,6 +122,7 @@ class Templar: self._filters = None self._tests = None self._available_variables = variables + self._cached_result = {} if loader: self._basedir = loader.get_basedir() @@ -298,18 +299,24 @@ class Templar: elif resolved_val is None: return C.DEFAULT_NULL_REPRESENTATION - result = self._do_template(variable, preserve_trailing_newlines=preserve_trailing_newlines, escape_backslashes=escape_backslashes, fail_on_undefined=fail_on_undefined, overrides=overrides) + # Using a cache in order to prevent template calls with already templated variables + cache_key = variable + str(preserve_trailing_newlines) + str(escape_backslashes) + str(overrides) + try: + result = self._cached_result[cache_key] + except KeyError: + result = self._do_template(variable, preserve_trailing_newlines=preserve_trailing_newlines, escape_backslashes=escape_backslashes, fail_on_undefined=fail_on_undefined, overrides=overrides) + if convert_data: + # if this looks like a dictionary or list, convert it to such using the safe_eval method + if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \ + result.startswith("[") or result in ("True", "False"): + eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True) + if eval_results[1] is None: + result = eval_results[0] + else: + # FIXME: if the safe_eval raised an error, should we do something with it? + pass + self._cached_result[cache_key] = result - if convert_data: - # if this looks like a dictionary or list, convert it to such using the safe_eval method - if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \ - result.startswith("[") or result in ("True", "False"): - eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True) - if eval_results[1] is None: - result = eval_results[0] - else: - # FIXME: if the safe_eval raised an error, should we do something with it? - pass #return self._clean_data(result) return result From 805f768dab8e01dcad91ea11f571a70b8ab64f79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannig=20Perr=C3=A9?= Date: Mon, 2 Nov 2015 07:57:44 +0100 Subject: [PATCH 2/7] Clear cache when updating template variables. --- lib/ansible/template/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 165b9e0e96..eded032c7c 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -260,6 +260,8 @@ class Templar: assert isinstance(variables, dict) self._available_variables = variables.copy() + # Clearing cache + self._cached_result = {} def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, convert_data=True): ''' From 4a8d1703d4d1522abafc52b643e249e8a830007f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannig=20Perr=C3=A9?= Date: Mon, 2 Nov 2015 21:04:20 +0100 Subject: [PATCH 3/7] New patch against hostvars.py. With this patch, Ansible run lose 50% of time. Little rewrite of previous patch to use sha1 signature. Use fail_on_undefined to compute sha1 signature. --- lib/ansible/template/__init__.py | 18 +++++++++++++----- lib/ansible/vars/hostvars.py | 18 ++++++++++++++++-- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index eded032c7c..1f41fcee7b 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -39,6 +39,11 @@ from ansible.template.template import AnsibleJ2Template from ansible.template.vars import AnsibleJ2Vars from ansible.utils.debug import debug +try: + from hashlib import sha1 +except ImportError: + from sha import sha as sha1 + from numbers import Number __all__ = ['Templar'] @@ -270,6 +275,9 @@ class Templar: before being sent through the template engine. ''' + if fail_on_undefined is None: + fail_on_undefined = self._fail_on_undefined_errors + # Don't template unsafe variables, instead drop them back down to # their constituent type. if hasattr(variable, '__UNSAFE__'): @@ -302,10 +310,10 @@ class Templar: return C.DEFAULT_NULL_REPRESENTATION # Using a cache in order to prevent template calls with already templated variables - cache_key = variable + str(preserve_trailing_newlines) + str(escape_backslashes) + str(overrides) - try: - result = self._cached_result[cache_key] - except KeyError: + sha1_hash = sha1(variable + str(preserve_trailing_newlines) + str(escape_backslashes) + str(fail_on_undefined) + str(overrides)).hexdigest() + if sha1_hash in self._cached_result: + result = self._cached_result[sha1_hash] + else: result = self._do_template(variable, preserve_trailing_newlines=preserve_trailing_newlines, escape_backslashes=escape_backslashes, fail_on_undefined=fail_on_undefined, overrides=overrides) if convert_data: # if this looks like a dictionary or list, convert it to such using the safe_eval method @@ -317,7 +325,7 @@ class Templar: else: # FIXME: if the safe_eval raised an error, should we do something with it? pass - self._cached_result[cache_key] = result + self._cached_result[sha1_hash] = result #return self._clean_data(result) diff --git a/lib/ansible/vars/hostvars.py b/lib/ansible/vars/hostvars.py index 9f83342be3..ff7fa84fb6 100644 --- a/lib/ansible/vars/hostvars.py +++ b/lib/ansible/vars/hostvars.py @@ -28,6 +28,11 @@ from ansible import constants as C from ansible.inventory.host import Host from ansible.template import Templar +try: + from hashlib import sha1 +except ImportError: + from sha import sha as sha1 + __all__ = ['HostVars'] # Note -- this is a Mapping, not a MutableMapping @@ -39,6 +44,7 @@ class HostVars(collections.Mapping): self._loader = loader self._play = play self._variable_manager = variable_manager + self._cached_result = {} hosts = inventory.get_hosts(ignore_limits_and_restrictions=True) @@ -68,8 +74,16 @@ class HostVars(collections.Mapping): host = self._lookup.get(host_name) data = self._variable_manager.get_vars(loader=self._loader, host=host, play=self._play, include_hostvars=False) - templar = Templar(variables=data, loader=self._loader) - return templar.template(data, fail_on_undefined=False) + + # Using cache in order to avoid template call + sha1_hash = sha1(str(data)).hexdigest() + if sha1_hash in self._cached_result: + result = self._cached_result[sha1_hash] + else: + templar = Templar(variables=data, loader=self._loader) + result = templar.template(data, fail_on_undefined=False) + self._cached_result[sha1_hash] = result + return result def __contains__(self, host_name): item = self.get(host_name) From 130139dc80c7b4353c9a4383c1e69dcd87a4f2d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannig=20Perr=C3=A9?= Date: Mon, 2 Nov 2015 21:32:49 +0100 Subject: [PATCH 4/7] Fix unicode issue introduced by previous commit. --- lib/ansible/template/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 1f41fcee7b..67a08f1644 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -310,7 +310,7 @@ class Templar: return C.DEFAULT_NULL_REPRESENTATION # Using a cache in order to prevent template calls with already templated variables - sha1_hash = sha1(variable + str(preserve_trailing_newlines) + str(escape_backslashes) + str(fail_on_undefined) + str(overrides)).hexdigest() + sha1_hash = sha1(variable.encode('utf-8') + str(preserve_trailing_newlines) + str(escape_backslashes) + str(fail_on_undefined) + str(overrides)).hexdigest() if sha1_hash in self._cached_result: result = self._cached_result[sha1_hash] else: From 30f827d92d0bb3b5090b93c9283af2d2db4f3431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannig=20Perr=C3=A9?= Date: Tue, 3 Nov 2015 11:52:09 +0100 Subject: [PATCH 5/7] Fix python3 test. --- lib/ansible/template/__init__.py | 4 +++- lib/ansible/vars/hostvars.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 67a08f1644..31d60a1d3d 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -310,7 +310,9 @@ class Templar: return C.DEFAULT_NULL_REPRESENTATION # Using a cache in order to prevent template calls with already templated variables - sha1_hash = sha1(variable.encode('utf-8') + str(preserve_trailing_newlines) + str(escape_backslashes) + str(fail_on_undefined) + str(overrides)).hexdigest() + variable_hash = sha1(text_type(variable).encode('utf-8')) + options_hash = sha1((text_type(preserve_trailing_newlines) + text_type(escape_backslashes) + text_type(fail_on_undefined) + text_type(overrides)).encode('utf-8')) + sha1_hash = variable_hash.hexdigest() + options_hash.hexdigest() if sha1_hash in self._cached_result: result = self._cached_result[sha1_hash] else: diff --git a/lib/ansible/vars/hostvars.py b/lib/ansible/vars/hostvars.py index ff7fa84fb6..bba46075ef 100644 --- a/lib/ansible/vars/hostvars.py +++ b/lib/ansible/vars/hostvars.py @@ -76,7 +76,7 @@ class HostVars(collections.Mapping): data = self._variable_manager.get_vars(loader=self._loader, host=host, play=self._play, include_hostvars=False) # Using cache in order to avoid template call - sha1_hash = sha1(str(data)).hexdigest() + sha1_hash = sha1(str(data).encode('utf-8')).hexdigest() if sha1_hash in self._cached_result: result = self._cached_result[sha1_hash] else: From e0aa3ff2326e0b90c0dfacfc8fb96a3b75956389 Mon Sep 17 00:00:00 2001 From: Yannig Perre Date: Wed, 4 Nov 2015 22:15:02 +0100 Subject: [PATCH 6/7] Cache against hosts pattern (fix a part of problem describe in https://github.com/ansible/ansible/issues/13023). --- lib/ansible/inventory/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index a967553385..710763a7f7 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -37,6 +37,8 @@ from ansible.plugins import vars_loader from ansible.utils.vars import combine_vars from ansible.parsing.utils.addresses import parse_address +HOSTS_PATTERNS_CACHE = {} + try: from __main__ import display except ImportError: @@ -156,6 +158,11 @@ class Inventory(object): or applied subsets """ + # Check if pattern already computed + pattern_hash = str(pattern) + if pattern_hash in HOSTS_PATTERNS_CACHE: + return HOSTS_PATTERNS_CACHE[pattern_hash] + patterns = Inventory.split_host_pattern(pattern) hosts = self._evaluate_patterns(patterns) @@ -170,6 +177,7 @@ class Inventory(object): if self._restriction is not None: hosts = [ h for h in hosts if h in self._restriction ] + HOSTS_PATTERNS_CACHE[pattern_hash] = hosts return hosts @classmethod From ccbdd6229a247e51cba15d8913687f03580a80ac Mon Sep 17 00:00:00 2001 From: Yannig Perre Date: Wed, 4 Nov 2015 22:16:14 +0100 Subject: [PATCH 7/7] Use static vars when computing host vars known to be static (inventory_hostname, inventory_dir etc.). --- lib/ansible/template/__init__.py | 7 +++++-- lib/ansible/vars/hostvars.py | 9 ++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 31d60a1d3d..8f9324752e 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -268,7 +268,7 @@ class Templar: # Clearing cache self._cached_result = {} - def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, convert_data=True): + def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, convert_data=True, static_vars = ['']): ''' Templates (possibly recursively) any given data as input. If convert_bare is set to True, the given data will be wrapped as a jinja2 variable ('{{foo}}') @@ -340,7 +340,10 @@ class Templar: # we don't use iteritems() here to avoid problems if the underlying dict # changes sizes due to the templating, which can happen with hostvars for k in variable.keys(): - d[k] = self.template(variable[k], preserve_trailing_newlines=preserve_trailing_newlines, fail_on_undefined=fail_on_undefined, overrides=overrides) + if k not in static_vars: + d[k] = self.template(variable[k], preserve_trailing_newlines=preserve_trailing_newlines, fail_on_undefined=fail_on_undefined, overrides=overrides) + else: + d[k] = variable[k] return d else: return variable diff --git a/lib/ansible/vars/hostvars.py b/lib/ansible/vars/hostvars.py index bba46075ef..b4cd5eeaf4 100644 --- a/lib/ansible/vars/hostvars.py +++ b/lib/ansible/vars/hostvars.py @@ -28,6 +28,13 @@ from ansible import constants as C from ansible.inventory.host import Host from ansible.template import Templar +STATIC_VARS = [ + 'inventory_hostname', 'inventory_hostname_short', + 'inventory_file', 'inventory_dir', 'playbook_dir', + 'ansible_play_hosts', 'play_hosts', 'groups', 'ungrouped', 'group_names', + 'ansible_version', 'omit', 'role_names' +] + try: from hashlib import sha1 except ImportError: @@ -81,7 +88,7 @@ class HostVars(collections.Mapping): result = self._cached_result[sha1_hash] else: templar = Templar(variables=data, loader=self._loader) - result = templar.template(data, fail_on_undefined=False) + result = templar.template(data, fail_on_undefined=False, static_vars=STATIC_VARS) self._cached_result[sha1_hash] = result return result