Make ready for data tagging (#9833)

* Fix dependent lookup.

* Fix ansible_type plugin utils and adjust documentation of reveal_ansible_type filter and ansible_type test.

* Fix diy callback plugin.

* Adjust to Data Tagging.

* Vendor and use internal code from ansible-core to fix YAML callback.

Ref: https://github.com/ansible/ansible/issues/84781

tests/integration/targets/callback_log_plays/runme.sh

@@ -17,5 +17,5 @@ ansible-playbook ping_log.yml -v "$@"
 # now force it to fail
 export ANSIBLE_LOG_FOLDER="logit.file"
 touch "${ANSIBLE_LOG_FOLDER}"
-ansible-playbook ping_log.yml -v "$@" 2>&1| grep 'Failure using method (v2_runner_on_ok) in callback plugin'
+ansible-playbook ping_log.yml -v "$@" 2>&1| grep -E "(Failure using method \(v2_runner_on_ok\) in callback plugin|Callback dispatch 'v2_runner_on_ok' failed for plugin)"
 [[ ! -f "${ANSIBLE_LOG_FOLDER}/localhost" ]]

tests/integration/targets/cmd_runner/action_plugins/_unsafe_assert.py

@@ -9,6 +9,12 @@ from ansible.errors import AnsibleError
 from ansible.playbook.conditional import Conditional
 from ansible.plugins.action import ActionBase
 
+try:
+    from ansible.utils.datatag import trust_value as _trust_value
+except ImportError:
+    def _trust_value(input):
+        return input
+
 
 class ActionModule(ActionBase):
     ''' Fail with custom message '''
@@ -36,12 +42,16 @@ class ActionModule(ActionBase):
 
         thats = self._task.args['that']
 
-        cond = Conditional(loader=self._loader)
         result['_ansible_verbose_always'] = True
 
         for that in thats:
-            cond.when = [str(self._make_safe(that))]
-            test_result = cond.evaluate_conditional(templar=self._templar, all_vars=task_vars)
+            if hasattr(self._templar, 'evaluate_conditional'):
+                trusted_that = _trust_value(that) if _trust_value else that
+                test_result = self._templar.evaluate_conditional(conditional=trusted_that)
+            else:
+                cond = Conditional(loader=self._loader)
+                cond.when = [str(self._make_safe(that))]
+                test_result = cond.evaluate_conditional(templar=self._templar, all_vars=task_vars)
             if not test_result:
                 result['failed'] = True
                 result['evaluated_to'] = test_result

tests/integration/targets/filter_reveal_ansible_type/tasks/tasks.yml

@@ -2,53 +2,60 @@
 # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# Substitution converts str to AnsibleUnicode
-# -------------------------------------------
+# Substitution converts str to AnsibleUnicode/_AnsibleTaggedStr
+# -------------------------------------------------------------
 
-- name: String. AnsibleUnicode.
+- name: String. AnsibleUnicode/_AnsibleTaggedStr.
   assert:
-    that: result == dtype
-    success_msg: '"abc" is {{ dtype }}'
-    fail_msg: '"abc" is {{ result }}'
+    that: result in dtype
+    success_msg: '"abc" is one of {{ dtype }}'
+    fail_msg: '"abc" is {{ result }}, not one of {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     data: "abc"
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'AnsibleUnicode'
+    dtype:
+      - 'AnsibleUnicode'
+      - '_AnsibleTaggedStr'
 
-- name: String. AnsibleUnicode alias str.
+- name: String. AnsibleUnicode/_AnsibleTaggedStr alias str.
   assert:
-    that: result == dtype
-    success_msg: '"abc" is {{ dtype }}'
-    fail_msg: '"abc" is {{ result }}'
+    that: result in dtype
+    success_msg: '"abc" is one of {{ dtype }}'
+    fail_msg: '"abc" is {{ result }}, not one of {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str"}
     data: "abc"
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
-    dtype: 'str'
+    dtype:
+      - 'str'
 
-- name: List. All items are AnsibleUnicode.
+- name: List. All items are AnsibleUnicode/_AnsibleTaggedStr.
   assert:
-    that: result == dtype
-    success_msg: '["a", "b", "c"] is {{ dtype }}'
-    fail_msg: '["a", "b", "c"] is {{ result }}'
+    that: result in dtype
+    success_msg: '["a", "b", "c"] is one of {{ dtype }}'
+    fail_msg: '["a", "b", "c"] is {{ result }}, not one of {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     data: ["a", "b", "c"]
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'list[AnsibleUnicode]'
+    dtype:
+      - 'list[AnsibleUnicode]'
+      - 'list[_AnsibleTaggedStr]'
 
-- name: Dictionary. All keys are AnsibleUnicode. All values are AnsibleUnicode.
+- name: Dictionary. All keys and values are AnsibleUnicode/_AnsibleTaggedStr.
   assert:
-    that: result == dtype
-    success_msg: '{"a": "foo", "b": "bar", "c": "baz"} is {{ dtype }}'
-    fail_msg: '{"a": "foo", "b": "bar", "c": "baz"} is {{ result }}'
+    that: result in dtype
+    success_msg: '{"a": "foo", "b": "bar", "c": "baz"} is one of {{ dtype }}'
+    fail_msg: '{"a": "foo", "b": "bar", "c": "baz"} is {{ result }}, not one of {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     data: {"a": "foo", "b": "bar", "c": "baz"}
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'dict[AnsibleUnicode, AnsibleUnicode]'
+    dtype:
+      - 'dict[AnsibleUnicode, AnsibleUnicode]'
+      - 'dict[_AnsibleTaggedStr, _AnsibleTaggedStr]'
 
 # No substitution and no alias. Type of strings is str
 # ----------------------------------------------------
@@ -57,7 +64,7 @@
   assert:
     that: result == dtype
     success_msg: '"abc" is {{ dtype }}'
-    fail_msg:  '"abc" is {{ result }}'
+    fail_msg:  '"abc" is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ "abc" | community.general.reveal_ansible_type }}'
@@ -67,7 +74,7 @@
   assert:
     that: result == dtype
     success_msg: '123 is {{ dtype }}'
-    fail_msg:  '123 is {{ result }}'
+    fail_msg:  '123 is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ 123 | community.general.reveal_ansible_type }}'
@@ -77,7 +84,7 @@
   assert:
     that: result == dtype
     success_msg: '123.45 is {{ dtype }}'
-    fail_msg:  '123.45 is {{ result }}'
+    fail_msg:  '123.45 is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ 123.45 | community.general.reveal_ansible_type }}'
@@ -87,7 +94,7 @@
   assert:
     that: result == dtype
     success_msg: 'true is {{ dtype }}'
-    fail_msg:  'true is {{ result }}'
+    fail_msg:  'true is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ true | community.general.reveal_ansible_type }}'
@@ -97,7 +104,7 @@
   assert:
     that: result == dtype
     success_msg: '["a", "b", "c"] is {{ dtype }}'
-    fail_msg:  '["a", "b", "c"] is {{ result }}'
+    fail_msg:  '["a", "b", "c"] is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ ["a", "b", "c"] | community.general.reveal_ansible_type }}'
@@ -107,7 +114,7 @@
   assert:
     that: result == dtype
     success_msg: '[{"a": 1}, {"b": 2}] is {{ dtype }}'
-    fail_msg:  '[{"a": 1}, {"b": 2}] is {{ result }}'
+    fail_msg:  '[{"a": 1}, {"b": 2}] is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ [{"a": 1}, {"b": 2}] | community.general.reveal_ansible_type }}'
@@ -117,7 +124,7 @@
   assert:
     that: result == dtype
     success_msg: '{"a": 1} is {{ dtype }}'
-    fail_msg:  '{"a": 1} is {{ result }}'
+    fail_msg:  '{"a": 1} is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ {"a": 1} | community.general.reveal_ansible_type }}'
@@ -127,23 +134,23 @@
   assert:
     that: result == dtype
     success_msg: '{"a": 1, "b": 2} is {{ dtype }}'
-    fail_msg:  '{"a": 1, "b": 2} is {{ result }}'
+    fail_msg:  '{"a": 1, "b": 2} is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
     result: '{{ {"a": 1, "b": 2} | community.general.reveal_ansible_type }}'
     dtype: dict[str, int]
 
-# Type of strings is AnsibleUnicode or str
-# ----------------------------------------
+# Type of strings is AnsibleUnicode/_AnsibleTaggedStr or str
+# ----------------------------------------------------------
 
 - name: Dictionary. The keys are integers or strings. All values are strings.
   assert:
     that: result == dtype
     success_msg: 'data is {{ dtype }}'
-    fail_msg: 'data is {{ result }}'
+    fail_msg: 'data is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str", "_AnsibleTaggedInt": "int"}
     data: {1: 'a', 'b': 'b'}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[int|str, str]
@@ -152,10 +159,10 @@
   assert:
     that: result == dtype
     success_msg: 'data is {{ dtype }}'
-    fail_msg: 'data is {{ result }}'
+    fail_msg: 'data is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str", "_AnsibleTaggedInt": "int"}
     data: {1: 'a', 2: 'b'}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[int, str]
@@ -164,10 +171,10 @@
   assert:
     that: result == dtype
     success_msg: 'data is {{ dtype }}'
-    fail_msg: 'data is {{ result }}'
+    fail_msg: 'data is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str", "_AnsibleTaggedInt": "int", "_AnsibleTaggedFloat": "float"}
     data: {'a': 1, 'b': 1.1, 'c': 'abc', 'd': True, 'e': ['x', 'y', 'z'], 'f': {'x': 1, 'y': 2}}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[str, bool|dict|float|int|list|str]
@@ -176,10 +183,10 @@
   assert:
     that: result == dtype
     success_msg: 'data is {{ dtype }}'
-    fail_msg: 'data is {{ result }}'
+    fail_msg: 'data is {{ result }}, not {{ dtype }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str", "_AnsibleTaggedInt": "int", "_AnsibleTaggedFloat": "float"}
     data: [1, 2, 1.1, 'abc', True, ['x', 'y', 'z'], {'x': 1, 'y': 2}]
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: list[bool|dict|float|int|list|str]

tests/integration/targets/module_helper/tasks/mdepfail.yml

@@ -8,11 +8,15 @@
   ignore_errors: true
   register: result
 
+- name: Show results
+  debug:
+    var: result
+
 - name: assert failing dependency
   assert:
     that:
       - result is failed
       - '"Failed to import" in result.msg'
       - '"nopackagewiththisname" in result.msg'
-      - '"ModuleNotFoundError:" in result.exception or "ImportError:" in result.exception'
-      - '"nopackagewiththisname" in result.exception'
+      - '"ModuleNotFoundError:" in result.exception or "ImportError:" in result.exception or "(traceback unavailable)" in result.exception'
+      - '"nopackagewiththisname" in result.exception or "(traceback unavailable)" in result.exception'

tests/integration/targets/module_helper/tasks/msimpleda.yml

@@ -3,15 +3,19 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 - set_fact:
-    attr2_d:
+    attr2_depr_dict:
       msg: Attribute attr2 is deprecated
       version: 9.9.9
       collection_name: community.general
-    attr2_d_29:
+    # With Data Tagging, the deprecation dict looks a bit different:
+    attr2_depr_dict_dt:
       msg: Attribute attr2 is deprecated
       version: 9.9.9
-- set_fact:
-    attr2_depr_dict: "{{ ((ansible_version.major, ansible_version.minor) < (2, 10))|ternary(attr2_d_29, attr2_d) }}"
+      plugin:
+        requested_name: msimpleda
+        resolved_name: msimpleda
+        type: module
+      collection_name: null  # should be "community.general"; this will hopefully change back because this seriously sucks
 
 - name: test msimpleda 1
   msimpleda:
@@ -23,17 +27,21 @@
     that:
       - simple1.a == 1
       - simple1.attr1 == "abc"
-      - ("deprecations" not in simple1) or attr2_depr_dict not in simple1.deprecations
+      - ("deprecations" not in simple1) or (attr2_depr_dict not in simple1.deprecations and attr2_depr_dict_dt not in simple1.deprecations)
 
 - name: test msimpleda 2
   msimpleda:
     a: 2
   register: simple2
 
+- name: Show results
+  debug:
+    var: simple2
+
 - name: assert simple2
   assert:
     that:
       - simple2.a == 2
       - simple2.attr2 == "def"
       - '"deprecations" in simple2'
-      - attr2_depr_dict in simple2.deprecations
+      - attr2_depr_dict in simple2.deprecations or attr2_depr_dict_dt in simple2.deprecations

tests/integration/targets/test_ansible_type/tasks/tasks.yml

@@ -14,7 +14,9 @@
   vars:
     data: "abc"
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'AnsibleUnicode'
+    dtype:
+      - 'AnsibleUnicode'
+      - '_AnsibleTaggedStr'
 
 - name: String. AnsibleUnicode alias str.
   assert:
@@ -23,7 +25,7 @@
     fail_msg: '"abc" is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias: {"AnsibleUnicode": "str", "_AnsibleTaggedStr": "str"}
     data: "abc"
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: 'str'
@@ -37,7 +39,9 @@
   vars:
     data: ["a", "b", "c"]
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'list[AnsibleUnicode]'
+    dtype:
+      - 'list[AnsibleUnicode]'
+      - 'list[_AnsibleTaggedStr]'
 
 - name: Dictionary. All keys are AnsibleUnicode. All values are AnsibleUnicode.
   assert:
@@ -48,7 +52,9 @@
   vars:
     data: {"a": "foo", "b": "bar", "c": "baz"}
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: 'dict[AnsibleUnicode, AnsibleUnicode]'
+    dtype:
+      - 'dict[AnsibleUnicode, AnsibleUnicode]'
+      - 'dict[_AnsibleTaggedStr, _AnsibleTaggedStr]'
 
 # No substitution and no alias. Type of strings is str
 # ----------------------------------------------------
@@ -143,7 +149,10 @@
     fail_msg: '{"1": "a", "b": "b"} is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias:
+      AnsibleUnicode: str
+      _AnsibleTaggedStr: str
+      _AnsibleTaggedInt: int
     data: {1: 'a', 'b': 'b'}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[int|str, str]
@@ -155,7 +164,10 @@
     fail_msg: '{"1": "a", "2": "b"} is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias:
+      AnsibleUnicode: str
+      _AnsibleTaggedStr: str
+      _AnsibleTaggedInt: int
     data: {1: 'a', 2: 'b'}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[int, str]
@@ -167,7 +179,11 @@
     fail_msg: '{"a": 1, "b": 1.1, "c": "abc", "d": true, "e": ["x", "y", "z"], "f": {"x": 1, "y": 2}} is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias:
+      AnsibleUnicode: str
+      _AnsibleTaggedStr: str
+      _AnsibleTaggedInt: int
+      _AnsibleTaggedFloat: float
     data: {'a': 1, 'b': 1.1, 'c': 'abc', 'd': True, 'e': ['x', 'y', 'z'], 'f': {'x': 1, 'y': 2}}
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: dict[str, bool|dict|float|int|list|str]
@@ -179,7 +195,11 @@
     fail_msg: '[1, 2, 1.1, "abc", true, ["x", "y", "z"], {"x": 1, "y": 2}] is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"AnsibleUnicode": "str"}
+    alias:
+      AnsibleUnicode: str
+      _AnsibleTaggedStr: str
+      _AnsibleTaggedInt: int
+      _AnsibleTaggedFloat: float
     data: [1, 2, 1.1, 'abc', True, ['x', 'y', 'z'], {'x': 1, 'y': 2}]
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: list[bool|dict|float|int|list|str]
@@ -196,7 +216,10 @@
   vars:
     data: abc
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: ['AnsibleUnicode', 'str']
+    dtype:
+      - 'AnsibleUnicode'
+      - '_AnsibleTaggedStr'
+      - 'str'
 
 - name: float or int
   assert:
@@ -207,7 +230,11 @@
   vars:
     data: 123
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: ['float', 'int']
+    dtype:
+      - 'float'
+      - 'int'
+      - '_AnsibleTaggedInt'
+      - '_AnsibleTaggedFloat'
 
 - name: float or int
   assert:
@@ -218,7 +245,11 @@
   vars:
     data: 123.45
     result: '{{ data | community.general.reveal_ansible_type }}'
-    dtype: ['float', 'int']
+    dtype:
+      - 'float'
+      - 'int'
+      - '_AnsibleTaggedInt'
+      - '_AnsibleTaggedFloat'
 
 # Multiple alias
 # --------------
@@ -230,7 +261,11 @@
     fail_msg: '123 is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"int": "number", "float": "number"}
+    alias:
+      int: number
+      float: number
+      _AnsibleTaggedInt: number
+      _AnsibleTaggedFloat: number
     data: 123
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: number
@@ -242,7 +277,11 @@
     fail_msg: '123.45 is {{ result }}'
     quiet: '{{ quiet_test | default(true) | bool }}'
   vars:
-    alias: {"int": "number", "float": "number"}
+    alias:
+      int: number
+      float: number
+      _AnsibleTaggedInt: number
+      _AnsibleTaggedFloat: number
     data: 123.45
     result: '{{ data | community.general.reveal_ansible_type(alias) }}'
     dtype: number