Merge fe31a04f3d into 70b5e362f9

.azure-pipelines/azure-pipelines.yml

@@ -71,6 +71,20 @@ stages:
             - test: 3
             - test: 4
             - test: extra
+  - stage: Sanity_datatagging
+    displayName: Sanity datatagging
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: Test {0}
+          testFormat: datatagging/sanity/{0}
+          targets:
+            - test: 1
+            - test: 2
+            - test: 3
+            - test: 4
+            - test: extra
   - stage: Sanity_2_18
     displayName: Sanity 2.18
     dependsOn: []
@@ -126,6 +140,21 @@ stages:
             - test: '3.11'
             - test: '3.12'
             - test: '3.13'
+  - stage: Units_datatagging
+    displayName: Units datatagging
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: Python {0}
+          testFormat: datatagging/units/{0}/1
+          targets:
+            - test: 3.8
+            - test: 3.9
+            - test: '3.10'
+            - test: '3.11'
+            - test: '3.12'
+            - test: '3.13'
   - stage: Units_2_18
     displayName: Units 2.18
     dependsOn: []
@@ -200,6 +229,26 @@ stages:
             - 1
             - 2
             - 3
+  - stage: Remote_datatagging
+    displayName: Remote datatagging
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: datatagging/{0}
+          targets:
+            - name: macOS 15.3
+              test: macos/15.3
+            - name: RHEL 9.5
+              test: rhel/9.5
+            - name: FreeBSD 14.2
+              test: freebsd/14.2
+            - name: FreeBSD 13.5
+              test: freebsd/13.5
+          groups:
+            - 1
+            - 2
+            - 3
   - stage: Remote_2_18
     displayName: Remote 2.18
     dependsOn: []
@@ -280,6 +329,26 @@ stages:
             - 1
             - 2
             - 3
+  - stage: Docker_datatagging
+    displayName: Docker datatagging
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: datatagging/linux/{0}
+          targets:
+            - name: Fedora 41
+              test: fedora41
+            - name: Alpine 3.21
+              test: alpine321
+            - name: Ubuntu 22.04
+              test: ubuntu2204
+            - name: Ubuntu 24.04
+              test: ubuntu2404
+          groups:
+            - 1
+            - 2
+            - 3
   - stage: Docker_2_18
     displayName: Docker 2.18
     dependsOn: []
@@ -409,19 +478,23 @@ stages:
   - stage: Summary
     condition: succeededOrFailed()
     dependsOn:
+      - Sanity_datatagging
       - Sanity_devel
       - Sanity_2_18
       - Sanity_2_17
       - Sanity_2_16
+      - Units_datatagging
       - Units_devel
       - Units_2_18
       - Units_2_17
       - Units_2_16
       - Remote_devel_extra_vms
+      - Remote_datatagging
       - Remote_devel
       - Remote_2_18
       - Remote_2_17
       - Remote_2_16
+      - Docker_datatagging
       - Docker_devel
       - Docker_2_18
       - Docker_2_17

.github/workflows/ansible-test.yml

@@ -76,7 +76,7 @@ jobs:
           pre-test-cmd: >-
             mkdir -p ../../ansible
             ;
-            git clone --depth=1 --single-branch https://github.com/ansible-collections/community.internal_test_tools.git ../../community/internal_test_tools
+            git clone --depth=1 --single-branch --branch ci-data-tagging https://github.com/felixfontein/community.internal_test_tools.git ../../community/internal_test_tools
           pull-request-change-detection: 'true'
           target-python-version: ${{ matrix.python }}
           testing-type: units
@@ -160,7 +160,7 @@ jobs:
             ;
             git clone --depth=1 --single-branch https://github.com/ansible-collections/community.docker.git ../../community/docker
             ;
-            git clone --depth=1 --single-branch https://github.com/ansible-collections/community.internal_test_tools.git ../../community/internal_test_tools
+            git clone --depth=1 --single-branch --branch ci-data-tagging https://github.com/felixfontein/community.internal_test_tools.git ../../community/internal_test_tools
           pull-request-change-detection: 'true'
           target: ${{ matrix.target }}
           target-python-version: ${{ matrix.python }}

changelogs/fragments/9833-data-tagging.yml

@@ -0,0 +1,9 @@
+bugfixes:
+  - "dependent look plugin - make compatible with ansible-core's Data Tagging feature (https://github.com/ansible-collections/community.general/pull/9833)."
+  - "reveal_ansible_type filter plugin and ansible_type test plugin - make compatible with ansible-core's Data Tagging feature (https://github.com/ansible-collections/community.general/pull/9833)."
+  - "diy callback plugin - make compatible with ansible-core's Data Tagging feature (https://github.com/ansible-collections/community.general/pull/9833)."
+  - "yaml callback plugin - use ansible-core internals to avoid breakage with Data Tagging (https://github.com/ansible-collections/community.general/pull/9833)."
+known_issues:
+  - "reveal_ansible_type filter plugin and ansible_type test plugin - note that ansible-core's Data Tagging feature implements new aliases,
+     such as ``_AnsibleTaggedStr`` for ``str``, ``_AnsibleTaggedInt`` for ``int``, and ``_AnsibleTaggedFloat`` for ``float``
+     (https://github.com/ansible-collections/community.general/pull/9833)."

plugins/callback/diy.py

@@ -785,6 +785,12 @@ from ansible.vars.manager import VariableManager
 from ansible.plugins.callback.default import CallbackModule as Default
 from ansible.module_utils.common.text.converters import to_text
 
+try:
+    from ansible.template import trust_as_template  # noqa: F401, pylint: disable=unused-import
+    SUPPORTS_DATA_TAGGING = True
+except ImportError:
+    SUPPORTS_DATA_TAGGING = False
+
 
 class DummyStdout(object):
     def flush(self):
@@ -838,7 +844,10 @@ class CallbackModule(Default):
         return _ret
 
     def _using_diy(self, spec):
-        return (spec['msg'] is not None) and (spec['msg'] != spec['vars']['omit'])
+        sentinel = object()
+        omit = spec['vars'].get('omit', sentinel)
+        # With Data Tagging, omit is sentinel
+        return (spec['msg'] is not None) and (spec['msg'] != omit or omit is sentinel)
 
     def _parent_has_callback(self):
         return hasattr(super(CallbackModule, self), sys._getframe(1).f_code.co_name)
@@ -894,7 +903,7 @@ class CallbackModule(Default):
             )
         _ret.update(_all)
 
-        _ret.update(_ret.get(self.DIY_NS, {self.DIY_NS: CallbackDIYDict()}))
+        _ret.update(_ret.get(self.DIY_NS, {self.DIY_NS: {} if SUPPORTS_DATA_TAGGING else CallbackDIYDict()}))
 
         _ret[self.DIY_NS].update({'playbook': {}})
         _playbook_attributes = ['entries', 'file_name', 'basedir']

plugins/callback/yaml.py

@@ -53,29 +53,77 @@ def should_use_block(value):
     return False
 
 
-class MyDumper(AnsibleDumper):
+try:
-    def represent_scalar(self, tag, value, style=None):
+    class MyDumper(AnsibleDumper):  # pylint: disable=inherit-non-class
-        """Uses block style for multi-line strings"""
+        def represent_scalar(self, tag, value, style=None):
-        if style is None:
+            """Uses block style for multi-line strings"""
-            if should_use_block(value):
+            if style is None:
-                style = '|'
+                if should_use_block(value):
-                # we care more about readable than accuracy, so...
+                    style = '|'
-                # ...no trailing space
+                    # we care more about readable than accuracy, so...
-                value = value.rstrip()
+                    # ...no trailing space
-                # ...and non-printable characters
+                    value = value.rstrip()
-                value = ''.join(x for x in value if x in string.printable or ord(x) >= 0xA0)
+                    # ...and non-printable characters
-                # ...tabs prevent blocks from expanding
+                    value = ''.join(x for x in value if x in string.printable or ord(x) >= 0xA0)
-                value = value.expandtabs()
+                    # ...tabs prevent blocks from expanding
-                # ...and odd bits of whitespace
+                    value = value.expandtabs()
-                value = re.sub(r'[\x0b\x0c\r]', '', value)
+                    # ...and odd bits of whitespace
-                # ...as does trailing space
+                    value = re.sub(r'[\x0b\x0c\r]', '', value)
-                value = re.sub(r' +\n', '\n', value)
+                    # ...as does trailing space
-            else:
+                    value = re.sub(r' +\n', '\n', value)
-                style = self.default_style
+                else:
-        node = yaml.representer.ScalarNode(tag, value, style=style)
+                    style = self.default_style
-        if self.alias_key is not None:
+            node = yaml.representer.ScalarNode(tag, value, style=style)
-            self.represented_objects[self.alias_key] = node
+            if self.alias_key is not None:
-        return node
+                self.represented_objects[self.alias_key] = node
+            return node
+except:  # noqa: E722, pylint: disable=bare-except
+    # This happens with Data Tagging, see https://github.com/ansible/ansible/issues/84781
+    # Until there is a better solution we'll resort to using ansible-core internals.
+    from ansible._internal._yaml import _dumper
+    import typing as t
+
+    class MyDumper(_dumper._BaseDumper):
+        # This code is mostly taken from ansible._internal._yaml._dumper
+        @classmethod
+        def _register_representers(cls) -> None:
+            cls.add_multi_representer(_dumper.AnsibleTaggedObject, cls.represent_ansible_tagged_object)
+            cls.add_multi_representer(_dumper.Tripwire, cls.represent_tripwire)
+            cls.add_multi_representer(_dumper.c.Mapping, _dumper.SafeRepresenter.represent_dict)
+            cls.add_multi_representer(_dumper.c.Sequence, _dumper.SafeRepresenter.represent_list)
+
+        def represent_ansible_tagged_object(self, data):
+            if ciphertext := _dumper.VaultHelper.get_ciphertext(data, with_tags=False):
+                return self.represent_scalar('!vault', ciphertext, style='|')
+
+            return self.represent_data(_dumper.AnsibleTagHelper.as_native_type(data))  # automatically decrypts encrypted strings
+
+        def represent_tripwire(self, data: _dumper.Tripwire) -> t.NoReturn:
+            data.trip()
+
+        # The following function is the same as in the try/except
+        def represent_scalar(self, tag, value, style=None):
+            """Uses block style for multi-line strings"""
+            if style is None:
+                if should_use_block(value):
+                    style = '|'
+                    # we care more about readable than accuracy, so...
+                    # ...no trailing space
+                    value = value.rstrip()
+                    # ...and non-printable characters
+                    value = ''.join(x for x in value if x in string.printable or ord(x) >= 0xA0)
+                    # ...tabs prevent blocks from expanding
+                    value = value.expandtabs()
+                    # ...and odd bits of whitespace
+                    value = re.sub(r'[\x0b\x0c\r]', '', value)
+                    # ...as does trailing space
+                    value = re.sub(r' +\n', '\n', value)
+                else:
+                    style = self.default_style
+            node = yaml.representer.ScalarNode(tag, value, style=style)
+            if self.alias_key is not None:
+                self.represented_objects[self.alias_key] = node
+            return node
 
 
 class CallbackModule(Default):

plugins/filter/reveal_ansible_type.py

@@ -23,29 +23,29 @@ options:
 """
 
 EXAMPLES = r"""
-# Substitution converts str to AnsibleUnicode
+# Substitution converts str to AnsibleUnicode or _AnsibleTaggedStr
-# -------------------------------------------
+# ----------------------------------------------------------------
 
-# String. AnsibleUnicode.
+# String. AnsibleUnicode or _AnsibleTaggedStr.
 - data: "abc"
   result: '{{ data | community.general.reveal_ansible_type }}'
-# result => AnsibleUnicode
+# result => AnsibleUnicode (or _AnsibleTaggedStr)
 
-# String. AnsibleUnicode alias str.
+# String. AnsibleUnicode/_AnsibleTaggedStr alias str.
-- alias: {"AnsibleUnicode": "str"}
+- alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str"}
   data: "abc"
   result: '{{ data | community.general.reveal_ansible_type(alias) }}'
 # result => str
 
-# List. All items are AnsibleUnicode.
+# List. All items are AnsibleUnicode/_AnsibleTaggedStr.
 - data: ["a", "b", "c"]
   result: '{{ data | community.general.reveal_ansible_type }}'
-# result => list[AnsibleUnicode]
+# result => list[AnsibleUnicode] or list[_AnsibleTaggedStr]
 
-# Dictionary. All keys are AnsibleUnicode. All values are AnsibleUnicode.
+# Dictionary. All keys and values are AnsibleUnicode/_AnsibleTaggedStr.
 - data: {"a": "foo", "b": "bar", "c": "baz"}
   result: '{{ data | community.general.reveal_ansible_type }}'
-# result => dict[AnsibleUnicode, AnsibleUnicode]
+# result => dict[AnsibleUnicode, AnsibleUnicode] or dict[_AnsibleTaggedStr, _AnsibleTaggedStr]
 
 # No substitution and no alias. Type of strings is str
 # ----------------------------------------------------
@@ -82,29 +82,43 @@ EXAMPLES = r"""
 - result: '{{ {"a": 1, "b": 2} | community.general.reveal_ansible_type }}'
 # result => dict[str, int]
 
-# Type of strings is AnsibleUnicode or str
+# Type of strings is AnsibleUnicode, _AnsibleTaggedStr, or str
-# ----------------------------------------
+# ------------------------------------------------------------
 
 # Dictionary. The keys are integers or strings. All values are strings.
-- alias: {"AnsibleUnicode": "str"}
+- alias:
+    AnsibleUnicode: str
+    _AnsibleTaggedStr: str
+    _AnsibleTaggedInt: int
   data: {1: 'a', 'b': 'b'}
   result: '{{ data | community.general.reveal_ansible_type(alias) }}'
 # result => dict[int|str, str]
 
 # Dictionary. All keys are integers. All values are keys.
-- alias: {"AnsibleUnicode": "str"}
+- alias:
+    AnsibleUnicode: str
+    _AnsibleTaggedStr: str
+    _AnsibleTaggedInt: int
   data: {1: 'a', 2: 'b'}
   result: '{{ data | community.general.reveal_ansible_type(alias) }}'
 # result => dict[int, str]
 
 # Dictionary. All keys are strings. Multiple types values.
-- alias: {"AnsibleUnicode": "str"}
+- alias:
+    AnsibleUnicode: str
+    _AnsibleTaggedStr: str
+    _AnsibleTaggedInt: int
+    _AnsibleTaggedFloat: float
   data: {'a': 1, 'b': 1.1, 'c': 'abc', 'd': true, 'e': ['x', 'y', 'z'], 'f': {'x': 1, 'y': 2}}
   result: '{{ data | community.general.reveal_ansible_type(alias) }}'
 # result => dict[str, bool|dict|float|int|list|str]
 
 # List. Multiple types items.
-- alias: {"AnsibleUnicode": "str"}
+- alias:
+    AnsibleUnicode: str
+    _AnsibleTaggedStr: str
+    _AnsibleTaggedInt: int
+    _AnsibleTaggedFloat: float
   data: [1, 2, 1.1, 'abc', true, ['x', 'y', 'z'], {'x': 1, 'y': 2}]
   result: '{{ data | community.general.reveal_ansible_type(alias) }}'
 # result => list[bool|dict|float|int|list|str]
@@ -122,6 +136,7 @@ from ansible_collections.community.general.plugins.plugin_utils.ansible_type imp
 def reveal_ansible_type(data, alias=None):
     """Returns data type"""
 
+    # TODO: expose use_native_type parameter
     return _ansible_type(data, alias)
 
 

plugins/lookup/dependent.py

@@ -130,12 +130,24 @@ from ansible.template import Templar
 
 from ansible_collections.community.general.plugins.module_utils.version import LooseVersion
 
+try:
+    from ansible.template import trust_as_template as _trust_as_template
+    HAS_DATATAGGING = True
+except ImportError:
+    HAS_DATATAGGING = False
+
 
 # Whether Templar has a cache, which can be controlled by Templar.template()'s cache option.
 # The cache was removed for ansible-core 2.14 (https://github.com/ansible/ansible/pull/78419)
 _TEMPLAR_HAS_TEMPLATE_CACHE = LooseVersion(ansible_version) < LooseVersion('2.14.0')
 
 
+def _make_safe(value):
+    if HAS_DATATAGGING and isinstance(value, str):
+        return _trust_as_template(value)
+    return value
+
+
 class LookupModule(LookupBase):
     def __evaluate(self, expression, templar, variables):
         """Evaluate expression with templar.
@@ -144,10 +156,13 @@ class LookupModule(LookupBase):
         ``variables`` are the variables to use.
         """
         templar.available_variables = variables or {}
-        expression = "{0}{1}{2}".format("{{", expression, "}}")
+        quoted_expression = "{0}{1}{2}".format("{{", expression, "}}")
         if _TEMPLAR_HAS_TEMPLATE_CACHE:
-            return templar.template(expression, cache=False)
+            return templar.template(quoted_expression, cache=False)
-        return templar.template(expression)
+        if hasattr(templar, 'evaluate_expression'):
+            # This is available since the Data Tagging PR has been merged
+            return templar.evaluate_expression(_make_safe(expression))
+        return templar.template(quoted_expression)
 
     def __process(self, result, terms, index, current, templar, variables):
         """Fills ``result`` list with evaluated items.

plugins/plugin_utils/ansible_type.py

@@ -8,17 +8,31 @@ __metaclass__ = type
 from ansible.errors import AnsibleFilterError
 from ansible.module_utils.common._collections_compat import Mapping
 
+try:
+    # Introduced with Data Tagging (https://github.com/ansible/ansible/pull/84621):
+    from ansible.module_utils.datatag import native_type_name as _native_type_name
+except ImportError:
+    _native_type_name = None
 
-def _atype(data, alias):
+
+def _atype(data, alias, *, use_native_type: bool = False):
     """
     Returns the name of the type class.
     """
 
-    data_type = type(data).__name__
+    if use_native_type and _native_type_name:
+        data_type = _native_type_name(data)
+    else:
+        data_type = type(data).__name__
+    # The following types were introduced with Data Tagging (https://github.com/ansible/ansible/pull/84621):
+    if data_type == "_AnsibleLazyTemplateDict":
+        data_type = "dict"
+    elif data_type == "_AnsibleLazyTemplateList":
+        data_type = "list"
     return alias.get(data_type, data_type)
 
 
-def _ansible_type(data, alias):
+def _ansible_type(data, alias, *, use_native_type: bool = False):
     """
     Returns the Ansible data type.
     """
@@ -30,16 +44,16 @@ def _ansible_type(data, alias):
         msg = "The argument alias must be a dictionary. %s is %s"
         raise AnsibleFilterError(msg % (alias, type(alias)))
 
-    data_type = _atype(data, alias)
+    data_type = _atype(data, alias, use_native_type=use_native_type)
 
     if data_type == 'list' and len(data) > 0:
-        items = [_atype(i, alias) for i in data]
+        items = [_atype(i, alias, use_native_type=use_native_type) for i in data]
         items_type = '|'.join(sorted(set(items)))
         return ''.join((data_type, '[', items_type, ']'))
 
     if data_type == 'dict' and len(data) > 0:
-        keys = [_atype(i, alias) for i in data.keys()]
+        keys = [_atype(i, alias, use_native_type=use_native_type) for i in data.keys()]
-        vals = [_atype(i, alias) for i in data.values()]
+        vals = [_atype(i, alias, use_native_type=use_native_type) for i in data.values()]
         keys_type = '|'.join(sorted(set(keys)))
         vals_type = '|'.join(sorted(set(vals)))
         return ''.join((data_type, '[', keys_type, ', ', vals_type, ']'))

plugins/test/ansible_type.py

@@ -28,30 +28,36 @@ DOCUMENTATION = '''
 
 EXAMPLES = '''
 
-# Substitution converts str to AnsibleUnicode
+# Substitution converts str to AnsibleUnicode or _AnsibleTaggedStr
-# -------------------------------------------
+# ----------------------------------------------------------------
 
-# String. AnsibleUnicode.
+# String. AnsibleUnicode or _AnsibleTaggedStr.
-dtype: AnsibleUnicode
+dtype:
+  - AnsibleUnicode
+  - _AnsibleTaggedStr
 data: "abc"
 result: '{{ data is community.general.ansible_type(dtype) }}'
 # result => true
 
-# String. AnsibleUnicode alias str.
+# String. AnsibleUnicode/_AnsibleTaggedStr alias str.
-alias: {"AnsibleUnicode": "str"}
+alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str"}
 dtype: str
 data: "abc"
 result: '{{ data is community.general.ansible_type(dtype, alias) }}'
 # result => true
 
-# List. All items are AnsibleUnicode.
+# List. All items are AnsibleUnicode/_AnsibleTaggedStr.
-dtype: list[AnsibleUnicode]
+dtype:
+  - list[AnsibleUnicode]
+  - list[_AnsibleTaggedStr]
 data: ["a", "b", "c"]
 result: '{{ data is community.general.ansible_type(dtype) }}'
 # result => true
 
-# Dictionary. All keys are AnsibleUnicode. All values are AnsibleUnicode.
+# Dictionary. All keys and values are AnsibleUnicode/_AnsibleTaggedStr.
-dtype: dict[AnsibleUnicode, AnsibleUnicode]
+dtype:
+  - dict[AnsibleUnicode, AnsibleUnicode]
+  - dict[_AnsibleTaggedStr, _AnsibleTaggedStr]
 data: {"a": "foo", "b": "bar", "c": "baz"}
 result: '{{ data is community.general.ansible_type(dtype) }}'
 # result => true
@@ -99,32 +105,46 @@ dtype: dict[str, int]
 result: '{{ {"a": 1, "b": 2} is community.general.ansible_type(dtype) }}'
 # result => true
 
-# Type of strings is AnsibleUnicode or str
+# Type of strings is AnsibleUnicode, _AnsibleTaggedStr, or str
-# ----------------------------------------
+# ------------------------------------------------------------
 
 # Dictionary. The keys are integers or strings. All values are strings.
-alias: {"AnsibleUnicode": "str"}
+alias:
+  AnsibleUnicode: str
+  _AnsibleTaggedStr: str
+  _AnsibleTaggedInt: int
 dtype: dict[int|str, str]
 data: {1: 'a', 'b': 'b'}
 result: '{{ data is community.general.ansible_type(dtype, alias) }}'
 # result => true
 
 # Dictionary. All keys are integers. All values are keys.
-alias: {"AnsibleUnicode": "str"}
+alias:
+  AnsibleUnicode: str
+  _AnsibleTaggedStr: str
+  _AnsibleTaggedInt: int
 dtype: dict[int, str]
 data: {1: 'a', 2: 'b'}
 result: '{{ data is community.general.ansible_type(dtype, alias) }}'
 # result => true
 
 # Dictionary. All keys are strings. Multiple types values.
-alias: {"AnsibleUnicode": "str"}
+alias:
+  AnsibleUnicode: str
+  _AnsibleTaggedStr: str
+  _AnsibleTaggedInt: int
+  _AnsibleTaggedFloat: float
 dtype: dict[str, bool|dict|float|int|list|str]
 data: {'a': 1, 'b': 1.1, 'c': 'abc', 'd': True, 'e': ['x', 'y', 'z'], 'f': {'x': 1, 'y': 2}}
 result: '{{ data is community.general.ansible_type(dtype, alias) }}'
 # result => true
 
 # List. Multiple types items.
-alias: {"AnsibleUnicode": "str"}
+alias:
+  AnsibleUnicode: str
+  _AnsibleTaggedStr: str
+  _AnsibleTaggedInt: int
+  _AnsibleTaggedFloat: float
 dtype: list[bool|dict|float|int|list|str]
 data: [1, 2, 1.1, 'abc', True, ['x', 'y', 'z'], {'x': 1, 'y': 2}]
 result: '{{ data is community.general.ansible_type(dtype, alias) }}'
@@ -133,20 +153,20 @@ result: '{{ data is community.general.ansible_type(dtype, alias) }}'
 # Option dtype is list
 # --------------------
 
-# AnsibleUnicode or str
+# AnsibleUnicode, _AnsibleTaggedStr, or str
-dtype: ['AnsibleUnicode', 'str']
+dtype: ['AnsibleUnicode', '_AnsibleTaggedStr', 'str']
 data: abc
 result: '{{ data is community.general.ansible_type(dtype) }}'
 # result => true
 
 # float or int
-dtype: ['float', 'int']
+dtype: ['float', 'int', "_AnsibleTaggedInt", "_AnsibleTaggedFloat"]
 data: 123
 result: '{{ data is community.general.ansible_type(dtype) }}'
 # result => true
 
 # float or int
-dtype: ['float', 'int']
+dtype: ['float', 'int', "_AnsibleTaggedInt", "_AnsibleTaggedFloat"]
 data: 123.45
 result: '{{ data is community.general.ansible_type(dtype) }}'
 # result => true
@@ -155,14 +175,22 @@ result: '{{ data is community.general.ansible_type(dtype) }}'
 # --------------
 
 # int alias number
-alias: {"int": "number", "float": "number"}
+alias:
+    int: number
+    float: number
+    _AnsibleTaggedInt: number
+    _AnsibleTaggedFloat: float
 dtype: number
 data: 123
 result: '{{ data is community.general.ansible_type(dtype, alias) }}'
 # result => true
 
 # float alias number
-alias: {"int": "number", "float": "number"}
+alias:
+    int: number
+    float: number
+    _AnsibleTaggedInt: number
+    _AnsibleTaggedFloat: float
 dtype: number
 data: 123.45
 result: '{{ data is community.general.ansible_type(dtype, alias) }}'
@@ -192,6 +220,7 @@ def ansible_type(data, dtype, alias=None):
     else:
         data_types = dtype
 
+    # TODO: expose use_native_type parameter
     return _ansible_type(data, alias) in data_types
 
 

tests/integration/targets/callback_log_plays/runme.sh

@@ -17,5 +17,5 @@ ansible-playbook ping_log.yml -v "$@"
 # now force it to fail
 export ANSIBLE_LOG_FOLDER="logit.file"
 touch "${ANSIBLE_LOG_FOLDER}"
-ansible-playbook ping_log.yml -v "$@" 2>&1| grep 'Failure using method (v2_runner_on_ok) in callback plugin'
+ansible-playbook ping_log.yml -v "$@" 2>&1| grep -E "(Failure using method \(v2_runner_on_ok\) in callback plugin|Callback dispatch 'v2_runner_on_ok' failed for plugin)"
 [[ ! -f "${ANSIBLE_LOG_FOLDER}/localhost" ]]

tests/integration/targets/cmd_runner/action_plugins/_unsafe_assert.py

@@ -9,6 +9,12 @@ from ansible.errors import AnsibleError
 from ansible.playbook.conditional import Conditional
 from ansible.plugins.action import ActionBase
 
+try:
+    from ansible.utils.datatag import trust_value as _trust_value
+except ImportError:
+    def _trust_value(input):
+        return input
+
 
 class ActionModule(ActionBase):
     ''' Fail with custom message '''
@@ -36,12 +42,16 @@ class ActionModule(ActionBase):
 
         thats = self._task.args['that']
 
-        cond = Conditional(loader=self._loader)
         result['_ansible_verbose_always'] = True
 
         for that in thats:
-            cond.when = [str(self._make_safe(that))]
+            if hasattr(self._templar, 'evaluate_conditional'):
-            test_result = cond.evaluate_conditional(templar=self._templar, all_vars=task_vars)
+                trusted_that = _trust_value(that) if _trust_value else that
+                test_result = self._templar.evaluate_conditional(conditional=trusted_that)
+            else:
+                cond = Conditional(loader=self._loader)
+                cond.when = [str(self._make_safe(that))]
+                test_result = cond.evaluate_conditional(templar=self._templar, all_vars=task_vars)
             if not test_result:
                 result['failed'] = True
                 result['evaluated_to'] = test_result

tests/integration/targets/filter_reveal_ansible_type/tasks/tasks.yml

@@ -2,53 +2,60 @@
 # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# Substitution converts str to AnsibleUnicode
+# Substitution converts str to AnsibleUnicode/_AnsibleTaggedStr
-# -------------------------------------------
+# -------------------------------------------------------------
 
-- name: String. AnsibleUnicode.
+- name: String. AnsibleUnicode/_AnsibleTaggedStr.
   assert:
-    that: result == dtype
+    that: result in dtype
-    success_msg: '"abc" is {{ dtype }}'
+    success_msg: '"abc" is one of {{ dtype }}'
-    fail_msg: '"abc" is {{ result }}'
+    fail_msg: '"abc" is {{ result }}, not one of {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     data: "abc"
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'AnsibleUnicode'
+    dtype:
+      - 'AnsibleUnicode'
+      - '_AnsibleTaggedStr'
 
-- name: String. AnsibleUnicode alias str.
+- name: String. AnsibleUnicode/_AnsibleTaggedStr alias str.
   assert:
-    that: result == dtype
+    that: result in dtype
-    success_msg: '"abc" is {{ dtype }}'
+    success_msg: '"abc" is one of {{ dtype }}'
-    fail_msg: '"abc" is {{ result }}'
+    fail_msg: '"abc" is {{ result }}, not one of {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str"}
     data: "abc"
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
-    dtype: 'str'
+    dtype:
+      - 'str'
 
-- name: List. All items are AnsibleUnicode.
+- name: List. All items are AnsibleUnicode/_AnsibleTaggedStr.
   assert:
-    that: result == dtype
+    that: result in dtype
-    success_msg: '["a", "b", "c"] is {{ dtype }}'
+    success_msg: '["a", "b", "c"] is one of {{ dtype }}'
-    fail_msg: '["a", "b", "c"] is {{ result }}'
+    fail_msg: '["a", "b", "c"] is {{ result }}, not one of {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     data: ["a", "b", "c"]
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'list[AnsibleUnicode]'
+    dtype:
+      - 'list[AnsibleUnicode]'
+      - 'list[_AnsibleTaggedStr]'
 
-- name: Dictionary. All keys are AnsibleUnicode. All values are AnsibleUnicode.
+- name: Dictionary. All keys and values are AnsibleUnicode/_AnsibleTaggedStr.
   assert:
-    that: result == dtype
+    that: result in dtype
-    success_msg: '{"a": "foo", "b": "bar", "c": "baz"} is {{ dtype }}'
+    success_msg: '{"a": "foo", "b": "bar", "c": "baz"} is one of {{ dtype }}'
-    fail_msg: '{"a": "foo", "b": "bar", "c": "baz"} is {{ result }}'
+    fail_msg: '{"a": "foo", "b": "bar", "c": "baz"} is {{ result }}, not one of {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     data: {"a": "foo", "b": "bar", "c": "baz"}
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'dict[AnsibleUnicode, AnsibleUnicode]'
+    dtype:
+      - 'dict[AnsibleUnicode, AnsibleUnicode]'
+      - 'dict[_AnsibleTaggedStr, _AnsibleTaggedStr]'
 
 # No substitution and no alias. Type of strings is str
 # ----------------------------------------------------
@@ -57,7 +64,7 @@
   assert:
     that: result == dtype
     success_msg: '"abc" is {{ dtype }}'
-    fail_msg:  '"abc" is {{ result }}'
+    fail_msg:  '"abc" is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ "abc" | community.general.reveal_ansible_type }}'
@@ -67,7 +74,7 @@
   assert:
     that: result == dtype
     success_msg: '123 is {{ dtype }}'
-    fail_msg:  '123 is {{ result }}'
+    fail_msg:  '123 is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ 123 | community.general.reveal_ansible_type }}'
@@ -77,7 +84,7 @@
   assert:
     that: result == dtype
     success_msg: '123.45 is {{ dtype }}'
-    fail_msg:  '123.45 is {{ result }}'
+    fail_msg:  '123.45 is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ 123.45 | community.general.reveal_ansible_type }}'
@@ -87,7 +94,7 @@
   assert:
     that: result == dtype
     success_msg: 'true is {{ dtype }}'
-    fail_msg:  'true is {{ result }}'
+    fail_msg:  'true is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ true | community.general.reveal_ansible_type }}'
@@ -97,7 +104,7 @@
   assert:
     that: result == dtype
     success_msg: '["a", "b", "c"] is {{ dtype }}'
-    fail_msg:  '["a", "b", "c"] is {{ result }}'
+    fail_msg:  '["a", "b", "c"] is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ ["a", "b", "c"] | community.general.reveal_ansible_type }}'
@@ -107,7 +114,7 @@
   assert:
     that: result == dtype
     success_msg: '[{"a": 1}, {"b": 2}] is {{ dtype }}'
-    fail_msg:  '[{"a": 1}, {"b": 2}] is {{ result }}'
+    fail_msg:  '[{"a": 1}, {"b": 2}] is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ [{"a": 1}, {"b": 2}] | community.general.reveal_ansible_type }}'
@@ -117,7 +124,7 @@
   assert:
     that: result == dtype
     success_msg: '{"a": 1} is {{ dtype }}'
-    fail_msg:  '{"a": 1} is {{ result }}'
+    fail_msg:  '{"a": 1} is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ {"a": 1} | community.general.reveal_ansible_type }}'
@@ -127,23 +134,23 @@
   assert:
     that: result == dtype
     success_msg: '{"a": 1, "b": 2} is {{ dtype }}'
-    fail_msg:  '{"a": 1, "b": 2} is {{ result }}'
+    fail_msg:  '{"a": 1, "b": 2} is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ {"a": 1, "b": 2} | community.general.reveal_ansible_type }}'
     dtype: dict[str, int]
 
-# Type of strings is AnsibleUnicode or str
+# Type of strings is AnsibleUnicode/_AnsibleTaggedStr or str
-# ----------------------------------------
+# ----------------------------------------------------------
 
 - name: Dictionary. The keys are integers or strings. All values are strings.
   assert:
     that: result == dtype
     success_msg: 'data is {{ dtype }}'
-    fail_msg: 'data is {{ result }}'
+    fail_msg: 'data is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str", "_AnsibleTaggedInt": "int"}
     data: {1: 'a', 'b': 'b'}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[int|str, str]
@@ -152,10 +159,10 @@
   assert:
     that: result == dtype
     success_msg: 'data is {{ dtype }}'
-    fail_msg: 'data is {{ result }}'
+    fail_msg: 'data is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str", "_AnsibleTaggedInt": "int"}
     data: {1: 'a', 2: 'b'}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[int, str]
@@ -164,10 +171,10 @@
   assert:
     that: result == dtype
     success_msg: 'data is {{ dtype }}'
-    fail_msg: 'data is {{ result }}'
+    fail_msg: 'data is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str", "_AnsibleTaggedInt": "int", "_AnsibleTaggedFloat": "float"}
     data: {'a': 1, 'b': 1.1, 'c': 'abc', 'd': True, 'e': ['x', 'y', 'z'], 'f': {'x': 1, 'y': 2}}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[str, bool|dict|float|int|list|str]
@@ -176,10 +183,10 @@
   assert:
     that: result == dtype
     success_msg: 'data is {{ dtype }}'
-    fail_msg: 'data is {{ result }}'
+    fail_msg: 'data is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str", "_AnsibleTaggedInt": "int", "_AnsibleTaggedFloat": "float"}
     data: [1, 2, 1.1, 'abc', True, ['x', 'y', 'z'], {'x': 1, 'y': 2}]
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: list[bool|dict|float|int|list|str]

tests/integration/targets/module_helper/tasks/mdepfail.yml

@@ -8,11 +8,15 @@
   ignore_errors: true
   register: result
 
+- name: Show results
+  debug:
+    var: result
+
 - name: assert failing dependency
   assert:
     that:
       - result is failed
       - '"Failed to import" in result.msg'
       - '"nopackagewiththisname" in result.msg'
-      - '"ModuleNotFoundError:" in result.exception or "ImportError:" in result.exception'
+      - '"ModuleNotFoundError:" in result.exception or "ImportError:" in result.exception or "(traceback unavailable)" in result.exception'
-      - '"nopackagewiththisname" in result.exception'
+      - '"nopackagewiththisname" in result.exception or "(traceback unavailable)" in result.exception'

tests/integration/targets/module_helper/tasks/msimpleda.yml

@@ -3,15 +3,19 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 - set_fact:
-    attr2_d:
+    attr2_depr_dict:
       msg: Attribute attr2 is deprecated
       version: 9.9.9
       collection_name: community.general
-    attr2_d_29:
+    # With Data Tagging, the deprecation dict looks a bit different:
+    attr2_depr_dict_dt:
       msg: Attribute attr2 is deprecated
       version: 9.9.9
-- set_fact:
+      plugin:
-    attr2_depr_dict: "{{ ((ansible_version.major, ansible_version.minor) < (2, 10))|ternary(attr2_d_29, attr2_d) }}"
+        requested_name: msimpleda
+        resolved_name: msimpleda
+        type: module
+      collection_name: null  # should be "community.general"; this will hopefully change back because this seriously sucks
 
 - name: test msimpleda 1
   msimpleda:
@@ -23,17 +27,21 @@
     that:
       - simple1.a == 1
       - simple1.attr1 == "abc"
-      - ("deprecations" not in simple1) or attr2_depr_dict not in simple1.deprecations
+      - ("deprecations" not in simple1) or (attr2_depr_dict not in simple1.deprecations and attr2_depr_dict_dt not in simple1.deprecations)
 
 - name: test msimpleda 2
   msimpleda:
     a: 2
   register: simple2
 
+- name: Show results
+  debug:
+    var: simple2
+
 - name: assert simple2
   assert:
     that:
       - simple2.a == 2
       - simple2.attr2 == "def"
       - '"deprecations" in simple2'
-      - attr2_depr_dict in simple2.deprecations
+      - attr2_depr_dict in simple2.deprecations or attr2_depr_dict_dt in simple2.deprecations

tests/integration/targets/test_ansible_type/tasks/tasks.yml

@@ -14,7 +14,9 @@
   vars:
     data: "abc"
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'AnsibleUnicode'
+    dtype:
+      - 'AnsibleUnicode'
+      - '_AnsibleTaggedStr'
 
 - name: String. AnsibleUnicode alias str.
   assert:
@@ -23,7 +25,7 @@
     fail_msg: '"abc" is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str"}
     data: "abc"
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: 'str'
@@ -37,7 +39,9 @@
   vars:
     data: ["a", "b", "c"]
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'list[AnsibleUnicode]'
+    dtype:
+      - 'list[AnsibleUnicode]'
+      - 'list[_AnsibleTaggedStr]'
 
 - name: Dictionary. All keys are AnsibleUnicode. All values are AnsibleUnicode.
   assert:
@@ -48,7 +52,9 @@
   vars:
     data: {"a": "foo", "b": "bar", "c": "baz"}
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'dict[AnsibleUnicode, AnsibleUnicode]'
+    dtype:
+      - 'dict[AnsibleUnicode, AnsibleUnicode]'
+      - 'dict[_AnsibleTaggedStr, _AnsibleTaggedStr]'
 
 # No substitution and no alias. Type of strings is str
 # ----------------------------------------------------
@@ -143,7 +149,10 @@
     fail_msg: '{"1": "a", "b": "b"} is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias:
+      AnsibleUnicode: str
+      _AnsibleTaggedStr: str
+      _AnsibleTaggedInt: int
     data: {1: 'a', 'b': 'b'}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[int|str, str]
@@ -155,7 +164,10 @@
     fail_msg: '{"1": "a", "2": "b"} is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias:
+      AnsibleUnicode: str
+      _AnsibleTaggedStr: str
+      _AnsibleTaggedInt: int
     data: {1: 'a', 2: 'b'}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[int, str]
@@ -167,7 +179,11 @@
     fail_msg: '{"a": 1, "b": 1.1, "c": "abc", "d": true, "e": ["x", "y", "z"], "f": {"x": 1, "y": 2}} is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias:
+      AnsibleUnicode: str
+      _AnsibleTaggedStr: str
+      _AnsibleTaggedInt: int
+      _AnsibleTaggedFloat: float
     data: {'a': 1, 'b': 1.1, 'c': 'abc', 'd': True, 'e': ['x', 'y', 'z'], 'f': {'x': 1, 'y': 2}}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[str, bool|dict|float|int|list|str]
@@ -179,7 +195,11 @@
     fail_msg: '[1, 2, 1.1, "abc", true, ["x", "y", "z"], {"x": 1, "y": 2}] is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias:
+      AnsibleUnicode: str
+      _AnsibleTaggedStr: str
+      _AnsibleTaggedInt: int
+      _AnsibleTaggedFloat: float
     data: [1, 2, 1.1, 'abc', True, ['x', 'y', 'z'], {'x': 1, 'y': 2}]
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: list[bool|dict|float|int|list|str]
@@ -196,7 +216,10 @@
   vars:
     data: abc
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: ['AnsibleUnicode', 'str']
+    dtype:
+      - 'AnsibleUnicode'
+      - '_AnsibleTaggedStr'
+      - 'str'
 
 - name: float or int
   assert:
@@ -207,7 +230,11 @@
   vars:
     data: 123
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: ['float', 'int']
+    dtype:
+      - 'float'
+      - 'int'
+      - '_AnsibleTaggedInt'
+      - '_AnsibleTaggedFloat'
 
 - name: float or int
   assert:
@@ -218,7 +245,11 @@
   vars:
     data: 123.45
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: ['float', 'int']
+    dtype:
+      - 'float'
+      - 'int'
+      - '_AnsibleTaggedInt'
+      - '_AnsibleTaggedFloat'
 
 # Multiple alias
 # --------------
@@ -230,7 +261,11 @@
     fail_msg: '123 is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"int": "number", "float": "number"}
+    alias:
+      int: number
+      float: number
+      _AnsibleTaggedInt: number
+      _AnsibleTaggedFloat: number
     data: 123
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: number
@@ -242,7 +277,11 @@
     fail_msg: '123.45 is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"int": "number", "float": "number"}
+    alias:
+      int: number
+      float: number
+      _AnsibleTaggedInt: number
+      _AnsibleTaggedFloat: number
     data: 123.45
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: number

tests/utils/shippable/shippable.sh

@@ -59,7 +59,9 @@ function retry
 command -v pip
 pip --version
 pip list --disable-pip-version-check
-if [ "${ansible_version}" == "devel" ]; then
+if [ "${ansible_version}" == "datatagging" ]; then
+    retry pip install https://github.com/ansible/ansible/archive/refs/pull/84621/head.tar.gz --disable-pip-version-check
+elif [ "${ansible_version}" == "devel" ]; then
     retry pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check
 else
     retry pip install "https://github.com/ansible/ansible/archive/stable-${ansible_version}.tar.gz" --disable-pip-version-check
@@ -75,7 +77,7 @@ fi
 
 # Nothing further should be added to this list.
 # This is to prevent modules or plugins in this collection having a runtime dependency on other collections.
-retry git clone --depth=1 --single-branch https://github.com/ansible-collections/community.internal_test_tools.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/internal_test_tools"
+retry git clone --depth=1 --single-branch --branch ci-data-tagging https://github.com/felixfontein/community.internal_test_tools.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/internal_test_tools"
 # NOTE: we're installing with git to work around Galaxy being a huge PITA (https://github.com/ansible/galaxy/issues/2429)
 # retry ansible-galaxy -vvv collection install community.internal_test_tools