Add filters to_yaml and to_nice_yaml (#10784)

* Add filters to_yaml and to_nice_yaml.

* Allow to redact sensitive values.

* Add basic tests.

* Work around https://github.com/ansible/ansible/issues/85783.

* Cleanup.

.github/BOTMETA.yml

@@ -209,6 +209,8 @@ files:
     maintainers: resmo
   $filters/to_months.yml:
     maintainers: resmo
+  $filters/to_nice_yaml.yml:
+    maintainers: felixfontein
   $filters/to_prettytable.py:
     maintainers: tgadiev
   $filters/to_seconds.yml:
@@ -217,6 +219,10 @@ files:
     maintainers: resmo
   $filters/to_weeks.yml:
     maintainers: resmo
+  $filters/to_yaml.py:
+    maintainers: felixfontein
+  $filters/to_yaml.yml:
+    maintainers: felixfontein
   $filters/to_years.yml:
     maintainers: resmo
   $filters/unicode_normalize.py:

plugins/filter/to_nice_yaml.yml

@@ -0,0 +1,89 @@
+# Copyright (c) Contributors to the Ansible project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+DOCUMENTATION:
+  name: to_nice_yaml
+  author:
+    - Ansible Core Team
+    - Felix Fontein (@felixfontein)
+  version_added: 11.3.0
+  short_description: Convert variable to YAML string
+  description:
+    - Converts an Ansible variable into a YAML string representation, without preserving vaulted strings as P(ansible.builtin.to_yaml#filter).
+    - This filter functions as a wrapper to the L(Python PyYAML library, https://pypi.org/project/PyYAML/)'s C(yaml.dump) function.
+  positional: _input
+  options:
+    _input:
+      description:
+        - A variable or expression that returns a data structure.
+      type: raw
+      required: true
+    indent:
+      description:
+        - Number of spaces to indent Python structures, mainly used for display to humans.
+      type: integer
+      default: 2
+    sort_keys:
+      description:
+        - Affects sorting of dictionary keys.
+      default: true
+      type: bool
+    default_style:
+      description:
+        - Indicates the style of the scalar.
+      choices:
+        - ''
+        - "'"
+        - '"'
+        - '|'
+        - '>'
+      type: string
+    canonical:
+      description:
+        - If set to V(true), export tag type to the output.
+      type: bool
+    width:
+      description:
+        - Set the preferred line width.
+      type: integer
+    line_break:
+      description:
+        - Specify the line break.
+      type: string
+    encoding:
+      description:
+        - Specify the output encoding.
+      type: string
+    explicit_start:
+      description:
+        - If set to V(true), adds an explicit start using C(---).
+      type: bool
+    explicit_end:
+      description:
+        - If set to V(true), adds an explicit end using C(...).
+      type: bool
+    redact_sensitive_values:
+      description:
+        - If set to V(true), vaulted strings are replaced by V(<redacted>) instead of being decrypted.
+        - With future ansible-core versions, this can extend to other strings tagged as sensitive.
+        - B(Note) that with ansible-core 2.18 and before this might not yield the expected result
+          since these versions of ansible-core strip the vault information away from strings that are
+          part of more complex data structures specified in C(vars).
+      type: bool
+      default: false
+  notes:
+    - More options may be available, see L(PyYAML documentation, https://pyyaml.org/wiki/PyYAMLDocumentation) for details.
+    - >-
+      These parameters to C(yaml.dump) are not accepted, as they are overridden internally: O(ignore:allow_unicode).
+
+EXAMPLES: |
+  ---
+  # Dump variable in a template to create a YAML document
+  value: "{{ github_workflow | community.general.to_nice_yaml }}"
+
+RETURN:
+  _value:
+    description:
+      - The YAML serialized string representing the variable structure inputted.
+    type: string

plugins/filter/to_yaml.py

@@ -0,0 +1,113 @@
+# Copyright (c) Contributors to the Ansible project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+
+import typing as t
+from collections.abc import Mapping, Set
+
+from yaml import dump
+try:
+    from yaml.cyaml import CSafeDumper as SafeDumper
+except ImportError:
+    from yaml import SafeDumper
+
+from ansible.module_utils.common.collections import is_sequence
+try:
+    # This is ansible-core 2.19+
+    from ansible.utils.vars import transform_to_native_types
+    from ansible.parsing.vault import VaultHelper, VaultLib
+except ImportError:
+    transform_to_native_types = None
+
+from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+from ansible.utils.unsafe_proxy import AnsibleUnsafe
+
+
+def _to_native_types_compat(value: t.Any, *, redact_value: str | None) -> t.Any:
+    """Compatibility function for ansible-core 2.18 and before."""
+    if value is None:
+        return value
+    if isinstance(value, AnsibleUnsafe):
+        # This only works up to ansible-core 2.18:
+        return _to_native_types_compat(value._strip_unsafe(), redact_value=redact_value)
+        # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway.
+    if isinstance(value, Mapping):
+        return {
+            _to_native_types_compat(key, redact_value=redact_value): _to_native_types_compat(val, redact_value=redact_value)
+            for key, val in value.items()
+        }
+    if isinstance(value, Set):
+        return {_to_native_types_compat(elt, redact_value=redact_value) for elt in value}
+    if is_sequence(value):
+        return [_to_native_types_compat(elt, redact_value=redact_value) for elt in value]
+    if isinstance(value, AnsibleVaultEncryptedUnicode):
+        if redact_value is not None:
+            return redact_value
+        # This only works up to ansible-core 2.18:
+        return value.data
+        # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway.
+    if isinstance(value, bytes):
+        return bytes(value)
+    if isinstance(value, str):
+        return str(value)
+
+    return value
+
+
+def _to_native_types(value: t.Any, *, redact: bool) -> t.Any:
+    if isinstance(value, Mapping):
+        return {_to_native_types(k, redact=redact): _to_native_types(v, redact=redact) for k, v in value.items()}
+    if is_sequence(value):
+        return [_to_native_types(e, redact=redact) for e in value]
+    if redact:
+        ciphertext = VaultHelper.get_ciphertext(value, with_tags=False)
+        if ciphertext and VaultLib.is_encrypted(ciphertext):
+            return "<redacted>"
+    return transform_to_native_types(value, redact=redact)
+
+
+def remove_all_tags(value: t.Any, *, redact_sensitive_values: bool = False) -> t.Any:
+    """
+    Remove all tags from all values in the input.
+
+    If ``redact_sensitive_values`` is ``True``, all sensitive values will be redacted.
+    """
+    if transform_to_native_types is not None:
+        return _to_native_types(value, redact=redact_sensitive_values)
+
+    return _to_native_types_compat(
+        value,
+        redact_value="<redacted>" if redact_sensitive_values else None,  # same string as in ansible-core 2.19 by transform_to_native_types()
+    )
+
+
+def to_yaml(value: t.Any, *, redact_sensitive_values: bool = False, default_flow_style: bool | None = None, **kwargs) -> str:
+    """Serialize input as terse flow-style YAML."""
+    return dump(
+        remove_all_tags(value, redact_sensitive_values=redact_sensitive_values),
+        Dumper=SafeDumper,
+        allow_unicode=True,
+        default_flow_style=default_flow_style,
+        **kwargs,
+    )
+
+
+def to_nice_yaml(value: t.Any, *, redact_sensitive_values: bool = False, indent: int = 2, default_flow_style: bool = False, **kwargs) -> str:
+    """Serialize input as verbose multi-line YAML."""
+    return to_yaml(
+        value,
+        redact_sensitive_values=redact_sensitive_values,
+        default_flow_style=default_flow_style,
+        indent=indent,
+        **kwargs,
+    )
+
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            'to_yaml': to_yaml,
+            'to_nice_yaml': to_nice_yaml,
+        }

plugins/filter/to_yaml.yml

@@ -0,0 +1,92 @@
+# Copyright (c) Contributors to the Ansible project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+DOCUMENTATION:
+  name: to_yaml
+  author:
+    - Ansible Core Team
+    - Felix Fontein (@felixfontein)
+  version_added: 11.3.0
+  short_description: Convert variable to YAML string
+  description:
+    - Converts an Ansible variable into a YAML string representation, without preserving vaulted strings as P(ansible.builtin.to_yaml#filter).
+    - This filter functions as a wrapper to the L(Python PyYAML library, https://pypi.org/project/PyYAML/)'s C(yaml.dump) function.
+  positional: _input
+  options:
+    _input:
+      description:
+        - A variable or expression that returns a data structure.
+      type: raw
+      required: true
+    indent:
+      description:
+        - Number of spaces to indent Python structures, mainly used for display to humans.
+      type: integer
+    sort_keys:
+      description:
+        - Affects sorting of dictionary keys.
+      default: true
+      type: bool
+    default_style:
+      description:
+        - Indicates the style of the scalar.
+      choices:
+        - ''
+        - "'"
+        - '"'
+        - '|'
+        - '>'
+      type: string
+    canonical:
+      description:
+        - If set to V(true), export tag type to the output.
+      type: bool
+    width:
+      description:
+        - Set the preferred line width.
+      type: integer
+    line_break:
+      description:
+        - Specify the line break.
+      type: string
+    encoding:
+      description:
+        - Specify the output encoding.
+      type: string
+    explicit_start:
+      description:
+        - If set to V(true), adds an explicit start using C(---).
+      type: bool
+    explicit_end:
+      description:
+        - If set to V(true), adds an explicit end using C(...).
+      type: bool
+    redact_sensitive_values:
+      description:
+        - If set to V(true), vaulted strings are replaced by V(<redacted>) instead of being decrypted.
+        - With future ansible-core versions, this can extend to other strings tagged as sensitive.
+        - B(Note) that with ansible-core 2.18 and before this might not yield the expected result
+          since these versions of ansible-core strip the vault information away from strings that are
+          part of more complex data structures specified in C(vars).
+      type: bool
+      default: false
+  notes:
+    - More options may be available, see L(PyYAML documentation, https://pyyaml.org/wiki/PyYAMLDocumentation) for details.
+    - >-
+      These parameters to C(yaml.dump) are not accepted, as they are overridden internally: O(ignore:allow_unicode).
+
+EXAMPLES: |
+  ---
+  # Dump variable in a template to create a YAML document
+  value: "{{ github_workflow | community.general.to_yaml }}"
+
+  ---
+  # Same as above but 'prettier' (equivalent to community.general.to_nice_yaml filter)
+  value: "{{ docker_config | community.general.to_yaml(indent=2) }}"
+
+RETURN:
+  _value:
+    description:
+      - The YAML serialized string representing the variable structure inputted.
+    type: string

tests/integration/targets/filter_to_yaml/aliases

@@ -0,0 +1,5 @@
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+azp/posix/3

tests/integration/targets/filter_to_yaml/main.yml

@@ -0,0 +1,188 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- hosts: localhost
+  gather_facts: false
+  vars_files:
+    - vaulted_vars.yml
+  vars:
+    timestamp: 2025-01-02T03:04:05Z
+    bar: foobarbaz
+  tasks:
+    - name: Print vaulted values
+      debug:
+        msg:
+          foo: "{{ foo }}"
+
+    - name: Convert values to YAML
+      set_fact:
+        vstr: "{{ 'foo' | community.general.to_yaml }}"
+        vvstr: "{{ foo | community.general.to_yaml }}"
+        vvstr_redact: "{{ foo | community.general.to_yaml(redact_sensitive_values=true) }}"
+        vint: "{{ 42 | community.general.to_yaml }}"
+        vfloat: "{{ -3.1415 | community.general.to_yaml }}"
+        vbool: "{{ true | community.general.to_yaml }}"
+        vtimestamp: "{{ timestamp | community.general.to_yaml }}"
+        vlist: "{{ [1, false, 'bar'] | community.general.to_yaml }}"
+        vdict: "{{ {'a': 'b', 1: 2} | community.general.to_yaml }}"
+
+    - name: Check values
+      assert:
+        that:
+          - 'vstr == "foo\n"'
+          - 'vvstr == "bar\n"'
+          - 'vvstr_redact == "<redacted>\n"'
+          - 'vint == "42\n"'
+          - 'vfloat == "-3.1415\n"'
+          - 'vbool == "true\n"'
+          - 'vtimestamp == "2025-01-02 03:04:05+00:00\n"'
+          - 'vlist == "[1, false, bar]\n"'
+          - 'vdict == "{a: b, 1: 2}\n"'
+
+    - name: Convert values to nice YAML
+      set_fact:
+        vstr: "{{ 'foo' | community.general.to_nice_yaml }}"
+        vvstr: "{{ foo | community.general.to_nice_yaml }}"
+        vvstr_redact: "{{ foo | community.general.to_nice_yaml(redact_sensitive_values=true) }}"
+        vint: "{{ 42 | community.general.to_nice_yaml }}"
+        vfloat: "{{ -3.1415 | community.general.to_nice_yaml }}"
+        vbool: "{{ true | community.general.to_nice_yaml }}"
+        vtimestamp: "{{ timestamp | community.general.to_nice_yaml }}"
+        vlist: "{{ [1, false, 'bar'] | community.general.to_nice_yaml }}"
+        vdict: "{{ {'a': 'b', 1: 2} | community.general.to_nice_yaml }}"
+
+    - name: Check values
+      assert:
+        that:
+          - 'vstr == "foo\n"'
+          - 'vvstr == "bar\n"'
+          - 'vvstr_redact == "<redacted>\n"'
+          - 'vint == "42\n"'
+          - 'vfloat == "-3.1415\n"'
+          - 'vbool == "true\n"'
+          - 'vtimestamp == "2025-01-02 03:04:05+00:00\n"'
+          - 'vlist == "- 1\n- false\n- bar\n"'
+          - 'vdict == "a: b\n1: 2\n"'
+
+    - name: Convert more complex data structure (from vars file)
+      set_fact:
+        complex: "{{ foobar | community.general.to_yaml }}"
+        complex_redact: "{{ foobar | community.general.to_yaml(redact_sensitive_values=true) }}"
+        complex_nice: "{{ foobar | community.general.to_nice_yaml }}"
+        complex_nice_redact: "{{ foobar | community.general.to_nice_yaml(redact_sensitive_values=true) }}"
+
+    - assert:
+        that:
+          - complex == exp_complex
+          - complex_redact == exp_complex_redact
+          - complex_nice == exp_complex_nice
+          - complex_nice_redact == exp_complex_nice_redact
+      vars:
+        exp_complex: |
+          a_list: [bar, ! '2025-02-03 04:05:06', Hello!, true, false]
+          a_value: 123
+        exp_complex_redact: |
+          a_list: [<redacted>, ! '2025-02-03 04:05:06', Hello!, true, false]
+          a_value: 123
+        exp_complex_nice: |
+          a_list:
+          - bar
+          - 2025-02-03 04:05:06
+          - Hello!
+          - true
+          - false
+          a_value: 123
+        exp_complex_nice_redact: |
+          a_list:
+          - <redacted>
+          - 2025-02-03 04:05:06
+          - Hello!
+          - true
+          - false
+          a_value: 123
+
+    - name: Convert more complex data structure (from vars)
+      set_fact:
+        complex: "{{ data | community.general.to_yaml }}"
+        complex_redact: "{{ data | community.general.to_yaml(redact_sensitive_values=true) }}"
+        complex_nice: "{{ data | community.general.to_nice_yaml }}"
+        complex_nice_redact: "{{ data | community.general.to_nice_yaml(redact_sensitive_values=true) }}"
+      vars:
+        data:
+          foo: 123
+          bar: 1.23
+          baz: true
+          bam: foobar
+          bang:
+            - "{{ timestamp }}"
+            - "{{ bar }}"
+            - "{{ foo }}"
+
+    - when: ansible_version.full is version("2.19", "<")
+      assert:
+        that:
+          - complex == exp_complex
+          # With ansible-core 2.18 and before, the vaulted string is decryped before it reaches the filter,
+          # so the redaction does not work there.
+          - complex_redact == exp_complex
+          - complex_nice == exp_complex_nice
+          # With ansible-core 2.18 and before, the vaulted string is decryped before it reaches the filter,
+          # so the redaction does not work there.
+          - complex_nice_redact == exp_complex_nice
+      vars:
+        exp_complex: |
+          bam: foobar
+          bang: ['2025-01-02 03:04:05+00:00', foobarbaz, bar]
+          bar: 1.23
+          baz: true
+          foo: 123
+        exp_complex_nice: |
+          bam: foobar
+          bang:
+          - '2025-01-02 03:04:05+00:00'
+          - foobarbaz
+          - bar
+          bar: 1.23
+          baz: true
+          foo: 123
+
+    - when: ansible_version.full is version("2.19", ">=")
+      assert:
+        that:
+          - complex == exp_complex
+          - complex_redact == exp_complex_redact
+          - complex_nice == exp_complex_nice
+          - complex_nice_redact == exp_complex_nice_redact
+      vars:
+        exp_complex: |
+          bam: foobar
+          bang: [! '2025-01-02 03:04:05+00:00', foobarbaz, bar]
+          bar: 1.23
+          baz: true
+          foo: 123
+        exp_complex_redact: |
+          bam: foobar
+          bang: [! '2025-01-02 03:04:05+00:00', foobarbaz, <redacted>]
+          bar: 1.23
+          baz: true
+          foo: 123
+        exp_complex_nice: |
+          bam: foobar
+          bang:
+          - 2025-01-02 03:04:05+00:00
+          - foobarbaz
+          - bar
+          bar: 1.23
+          baz: true
+          foo: 123
+        exp_complex_nice_redact: |
+          bam: foobar
+          bang:
+          - 2025-01-02 03:04:05+00:00
+          - foobarbaz
+          - <redacted>
+          bar: 1.23
+          baz: true
+          foo: 123

tests/integration/targets/filter_to_yaml/password

@@ -0,0 +1 @@
+secret

tests/integration/targets/filter_to_yaml/password.license

@@ -0,0 +1,3 @@
+Copyright (c) Ansible Project
+GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+SPDX-License-Identifier: GPL-3.0-or-later

tests/integration/targets/filter_to_yaml/runme.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+set -eux
+
+ansible-playbook --vault-password-file password main.yml "$@"

tests/integration/targets/filter_to_yaml/vaulted_vars.yml

@@ -0,0 +1,27 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+foo: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  32336431346561346535396563363438333131636539653331376466383331663838303835353862
+  3536306130663166393533626530646435383938323066320a303366613035323835373030303262
+  35633636653362393531653961396665663965356562346538643863336562393734376234313134
+  3562663234326435390a376464633234373636643538353562326133316439343863373333363265
+  6239
+
+foobar:
+  a_value: 123
+  a_list:
+    - !vault |
+      $ANSIBLE_VAULT;1.1;AES256
+      32336431346561346535396563363438333131636539653331376466383331663838303835353862
+      3536306130663166393533626530646435383938323066320a303366613035323835373030303262
+      35633636653362393531653961396665663965356562346538643863336562393734376234313134
+      3562663234326435390a376464633234373636643538353562326133316439343863373333363265
+      6239
+    - 2025-02-03 04:05:06
+    - Hello!
+    - true
+    - false