Allow config to enable native jinja types (#32738)

Co-authored-by: Martin Krizek <martin.krizek@gmail.com>

lib/ansible/config/base.yml

@@ -327,7 +327,7 @@ LOCALHOST_WARNING:
   description:
     - By default Ansible will issue a warning when there are no hosts in the
       inventory.
-    - These warnings can be silenced by adjusting this setting to False. 
+    - These warnings can be silenced by adjusting this setting to False.
   env: [{name: ANSIBLE_LOCALHOST_WARNING}]
   ini:
   - {key: localhost_warning, section: defaults}
@@ -508,7 +508,7 @@ DEFAULT_DEBUG:
   description:
     - "Toggles debug output in Ansible. This is *very* verbose and can hinder
       multiprocessing.  Debug output can also include secret information
-      despite no_log settings being enabled, which means debug mode should not be used in 
+      despite no_log settings being enabled, which means debug mode should not be used in
       production."
   env: [{name: ANSIBLE_DEBUG}]
   ini:
@@ -694,6 +694,16 @@ DEFAULT_JINJA2_EXTENSIONS:
   env: [{name: ANSIBLE_JINJA2_EXTENSIONS}]
   ini:
   - {key: jinja2_extensions, section: defaults}
+DEFAULT_JINJA2_NATIVE:
+  name: Use Jinja2's NativeEnvironment for templating
+  default: False
+  description: This option preserves variable types during template operations. This requires Jinja2 >= 2.10.
+  env: [{name: ANSIBLE_JINJA2_NATIVE}]
+  ini:
+  - {key: jinja2_native, section: defaults}
+  type: boolean
+  yaml: {key: jinja2_native}
+  version_added: 2.7
 DEFAULT_KEEP_REMOTE_FILES:
   name: Keep remote files
   default: False

lib/ansible/template/__init__.py

@@ -37,11 +37,9 @@ try:
 except ImportError:
     from sha import sha as sha1
 
-from jinja2 import Environment
 from jinja2.exceptions import TemplateSyntaxError, UndefinedError
 from jinja2.loaders import FileSystemLoader
 from jinja2.runtime import Context, StrictUndefined
-from jinja2.utils import concat as j2_concat
 
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable, AnsibleAssertionError
@@ -70,6 +68,19 @@ NON_TEMPLATED_TYPES = (bool, Number)
 
 JINJA2_OVERRIDE = '#jinja2:'
 
+USE_JINJA2_NATIVE = False
+if C.DEFAULT_JINJA2_NATIVE:
+    try:
+        from jinja2.nativetypes import NativeEnvironment as Environment
+        from ansible.template.native_helpers import ansible_native_concat as j2_concat
+        USE_JINJA2_NATIVE = True
+    except ImportError:
+        from jinja2 import Environment
+        from jinja2.utils import concat as j2_concat
+else:
+    from jinja2 import Environment
+    from jinja2.utils import concat as j2_concat
+
 
 def generate_ansible_template_vars(path):
     b_path = to_bytes(path)
@@ -479,19 +490,20 @@ class Templar:
                             disable_lookups=disable_lookups,
                         )
 
-                        unsafe = hasattr(result, '__UNSAFE__')
-                        if convert_data and not self._no_type_regex.match(variable):
-                            # if this looks like a dictionary or list, convert it to such using the safe_eval method
-                            if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \
-                                    result.startswith("[") or result in ("True", "False"):
-                                eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True)
-                                if eval_results[1] is None:
-                                    result = eval_results[0]
-                                    if unsafe:
-                                        result = wrap_var(result)
-                                else:
-                                    # FIXME: if the safe_eval raised an error, should we do something with it?
-                                    pass
+                        if not USE_JINJA2_NATIVE:
+                            unsafe = hasattr(result, '__UNSAFE__')
+                            if convert_data and not self._no_type_regex.match(variable):
+                                # if this looks like a dictionary or list, convert it to such using the safe_eval method
+                                if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \
+                                        result.startswith("[") or result in ("True", "False"):
+                                    eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True)
+                                    if eval_results[1] is None:
+                                        result = eval_results[0]
+                                        if unsafe:
+                                            result = wrap_var(result)
+                                    else:
+                                        # FIXME: if the safe_eval raised an error, should we do something with it?
+                                        pass
 
                         # we only cache in the case where we have a single variable
                         # name, to make sure we're not putting things which may otherwise
@@ -663,9 +675,15 @@ class Templar:
             raise AnsibleError("lookup plugin (%s) not found" % name)
 
     def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, disable_lookups=False):
+        if USE_JINJA2_NATIVE and not isinstance(data, string_types):
+            return data
+
         # For preserving the number of input newlines in the output (used
         # later in this method)
-        data_newlines = _count_newlines_from_end(data)
+        if not USE_JINJA2_NATIVE:
+            data_newlines = _count_newlines_from_end(data)
+        else:
+            data_newlines = None
 
         if fail_on_undefined is None:
             fail_on_undefined = self._fail_on_undefined_errors
@@ -678,7 +696,7 @@ class Templar:
                 myenv = self.environment.overlay(overrides)
 
             # Get jinja env overrides from template
-            if data.startswith(JINJA2_OVERRIDE):
+            if hasattr(data, 'startswith') and data.startswith(JINJA2_OVERRIDE):
                 eol = data.find('\n')
                 line = data[len(JINJA2_OVERRIDE):eol]
                 data = data[eol + 1:]
@@ -720,7 +738,7 @@ class Templar:
 
             try:
                 res = j2_concat(rf)
-                if new_context.unsafe:
+                if getattr(new_context, 'unsafe', False):
                     res = wrap_var(res)
             except TypeError as te:
                 if 'StrictUndefined' in to_native(te):
@@ -731,6 +749,9 @@ class Templar:
                     display.debug("failing because of a type error, template data is: %s" % to_native(data))
                     raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data), to_native(te)))
 
+            if USE_JINJA2_NATIVE:
+                return res
+
             if preserve_trailing_newlines:
                 # The low level calls above do not preserve the newline
                 # characters at the end of the input data, so we use the

lib/ansible/template/native_helpers.py

@@ -0,0 +1,44 @@
+# Copyright: (c) 2018, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+from ast import literal_eval
+from itertools import islice, chain
+import types
+
+from jinja2._compat import text_type
+
+
+def ansible_native_concat(nodes):
+    """Return a native Python type from the list of compiled nodes. If the
+    result is a single node, its value is returned. Otherwise, the nodes are
+    concatenated as strings. If the result can be parsed with
+    :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the
+    string is returned.
+    """
+
+    # https://github.com/pallets/jinja/blob/master/jinja2/nativetypes.py
+
+    head = list(islice(nodes, 2))
+
+    if not head:
+        return None
+
+    if len(head) == 1:
+        out = head[0]
+        # short circuit literal_eval when possible
+        if not isinstance(out, list):  # FIXME is this needed?
+            return out
+    else:
+        if isinstance(nodes, types.GeneratorType):
+            nodes = chain(head, nodes)
+        out = u''.join([text_type(v) for v in nodes])
+
+    try:
+        return literal_eval(out)
+    except (ValueError, SyntaxError, MemoryError):
+        return out