From 5fca1f641bf0bfe61b36aba3dc83e1f8d0fedcc4 Mon Sep 17 00:00:00 2001 From: "patchback[bot]" <45432694+patchback[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 18:54:47 +0200 Subject: [PATCH] [PR #10784/062b63bd backport][stable-11] Add filters to_yaml and to_nice_yaml (#10802) Add filters to_yaml and to_nice_yaml (#10784) * Add filters to_yaml and to_nice_yaml. * Allow to redact sensitive values. * Add basic tests. * Work around https://github.com/ansible/ansible/issues/85783. * Cleanup. (cherry picked from commit 062b63bda5bec8a662c60b3a63029bc5fbc50bec) Co-authored-by: Felix Fontein --- .github/BOTMETA.yml | 6 + plugins/filter/to_nice_yaml.yml | 89 +++++++++ plugins/filter/to_yaml.py | 113 +++++++++++ plugins/filter/to_yaml.yml | 92 +++++++++ .../targets/filter_to_yaml/aliases | 5 + .../targets/filter_to_yaml/main.yml | 188 ++++++++++++++++++ .../targets/filter_to_yaml/password | 1 + .../targets/filter_to_yaml/password.license | 3 + .../targets/filter_to_yaml/runme.sh | 8 + .../targets/filter_to_yaml/vaulted_vars.yml | 27 +++ 10 files changed, 532 insertions(+) create mode 100644 plugins/filter/to_nice_yaml.yml create mode 100644 plugins/filter/to_yaml.py create mode 100644 plugins/filter/to_yaml.yml create mode 100644 tests/integration/targets/filter_to_yaml/aliases create mode 100644 tests/integration/targets/filter_to_yaml/main.yml create mode 100644 tests/integration/targets/filter_to_yaml/password create mode 100644 tests/integration/targets/filter_to_yaml/password.license create mode 100755 tests/integration/targets/filter_to_yaml/runme.sh create mode 100644 tests/integration/targets/filter_to_yaml/vaulted_vars.yml diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index 8744ca0567..b498cdf9ea 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -209,6 +209,8 @@ files: maintainers: resmo $filters/to_months.yml: maintainers: resmo + $filters/to_nice_yaml.yml: + maintainers: felixfontein $filters/to_prettytable.py: maintainers: tgadiev $filters/to_seconds.yml: @@ -217,6 +219,10 @@ files: maintainers: resmo $filters/to_weeks.yml: maintainers: resmo + $filters/to_yaml.py: + maintainers: felixfontein + $filters/to_yaml.yml: + maintainers: felixfontein $filters/to_years.yml: maintainers: resmo $filters/unicode_normalize.py: diff --git a/plugins/filter/to_nice_yaml.yml b/plugins/filter/to_nice_yaml.yml new file mode 100644 index 0000000000..fe7a316f46 --- /dev/null +++ b/plugins/filter/to_nice_yaml.yml @@ -0,0 +1,89 @@ +# Copyright (c) Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +DOCUMENTATION: + name: to_nice_yaml + author: + - Ansible Core Team + - Felix Fontein (@felixfontein) + version_added: 11.3.0 + short_description: Convert variable to YAML string + description: + - Converts an Ansible variable into a YAML string representation, without preserving vaulted strings as P(ansible.builtin.to_yaml#filter). + - This filter functions as a wrapper to the L(Python PyYAML library, https://pypi.org/project/PyYAML/)'s C(yaml.dump) function. + positional: _input + options: + _input: + description: + - A variable or expression that returns a data structure. + type: raw + required: true + indent: + description: + - Number of spaces to indent Python structures, mainly used for display to humans. + type: integer + default: 2 + sort_keys: + description: + - Affects sorting of dictionary keys. + default: true + type: bool + default_style: + description: + - Indicates the style of the scalar. + choices: + - '' + - "'" + - '"' + - '|' + - '>' + type: string + canonical: + description: + - If set to V(true), export tag type to the output. + type: bool + width: + description: + - Set the preferred line width. + type: integer + line_break: + description: + - Specify the line break. + type: string + encoding: + description: + - Specify the output encoding. + type: string + explicit_start: + description: + - If set to V(true), adds an explicit start using C(---). + type: bool + explicit_end: + description: + - If set to V(true), adds an explicit end using C(...). + type: bool + redact_sensitive_values: + description: + - If set to V(true), vaulted strings are replaced by V() instead of being decrypted. + - With future ansible-core versions, this can extend to other strings tagged as sensitive. + - B(Note) that with ansible-core 2.18 and before this might not yield the expected result + since these versions of ansible-core strip the vault information away from strings that are + part of more complex data structures specified in C(vars). + type: bool + default: false + notes: + - More options may be available, see L(PyYAML documentation, https://pyyaml.org/wiki/PyYAMLDocumentation) for details. + - >- + These parameters to C(yaml.dump) are not accepted, as they are overridden internally: O(ignore:allow_unicode). + +EXAMPLES: | + --- + # Dump variable in a template to create a YAML document + value: "{{ github_workflow | community.general.to_nice_yaml }}" + +RETURN: + _value: + description: + - The YAML serialized string representing the variable structure inputted. + type: string diff --git a/plugins/filter/to_yaml.py b/plugins/filter/to_yaml.py new file mode 100644 index 0000000000..905b04271c --- /dev/null +++ b/plugins/filter/to_yaml.py @@ -0,0 +1,113 @@ +# Copyright (c) Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import annotations + +import typing as t +from collections.abc import Mapping, Set + +from yaml import dump +try: + from yaml.cyaml import CSafeDumper as SafeDumper +except ImportError: + from yaml import SafeDumper + +from ansible.module_utils.common.collections import is_sequence +try: + # This is ansible-core 2.19+ + from ansible.utils.vars import transform_to_native_types + from ansible.parsing.vault import VaultHelper, VaultLib +except ImportError: + transform_to_native_types = None + +from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode +from ansible.utils.unsafe_proxy import AnsibleUnsafe + + +def _to_native_types_compat(value: t.Any, *, redact_value: str | None) -> t.Any: + """Compatibility function for ansible-core 2.18 and before.""" + if value is None: + return value + if isinstance(value, AnsibleUnsafe): + # This only works up to ansible-core 2.18: + return _to_native_types_compat(value._strip_unsafe(), redact_value=redact_value) + # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway. + if isinstance(value, Mapping): + return { + _to_native_types_compat(key, redact_value=redact_value): _to_native_types_compat(val, redact_value=redact_value) + for key, val in value.items() + } + if isinstance(value, Set): + return {_to_native_types_compat(elt, redact_value=redact_value) for elt in value} + if is_sequence(value): + return [_to_native_types_compat(elt, redact_value=redact_value) for elt in value] + if isinstance(value, AnsibleVaultEncryptedUnicode): + if redact_value is not None: + return redact_value + # This only works up to ansible-core 2.18: + return value.data + # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway. + if isinstance(value, bytes): + return bytes(value) + if isinstance(value, str): + return str(value) + + return value + + +def _to_native_types(value: t.Any, *, redact: bool) -> t.Any: + if isinstance(value, Mapping): + return {_to_native_types(k, redact=redact): _to_native_types(v, redact=redact) for k, v in value.items()} + if is_sequence(value): + return [_to_native_types(e, redact=redact) for e in value] + if redact: + ciphertext = VaultHelper.get_ciphertext(value, with_tags=False) + if ciphertext and VaultLib.is_encrypted(ciphertext): + return "" + return transform_to_native_types(value, redact=redact) + + +def remove_all_tags(value: t.Any, *, redact_sensitive_values: bool = False) -> t.Any: + """ + Remove all tags from all values in the input. + + If ``redact_sensitive_values`` is ``True``, all sensitive values will be redacted. + """ + if transform_to_native_types is not None: + return _to_native_types(value, redact=redact_sensitive_values) + + return _to_native_types_compat( + value, + redact_value="" if redact_sensitive_values else None, # same string as in ansible-core 2.19 by transform_to_native_types() + ) + + +def to_yaml(value: t.Any, *, redact_sensitive_values: bool = False, default_flow_style: bool | None = None, **kwargs) -> str: + """Serialize input as terse flow-style YAML.""" + return dump( + remove_all_tags(value, redact_sensitive_values=redact_sensitive_values), + Dumper=SafeDumper, + allow_unicode=True, + default_flow_style=default_flow_style, + **kwargs, + ) + + +def to_nice_yaml(value: t.Any, *, redact_sensitive_values: bool = False, indent: int = 2, default_flow_style: bool = False, **kwargs) -> str: + """Serialize input as verbose multi-line YAML.""" + return to_yaml( + value, + redact_sensitive_values=redact_sensitive_values, + default_flow_style=default_flow_style, + indent=indent, + **kwargs, + ) + + +class FilterModule(object): + def filters(self): + return { + 'to_yaml': to_yaml, + 'to_nice_yaml': to_nice_yaml, + } diff --git a/plugins/filter/to_yaml.yml b/plugins/filter/to_yaml.yml new file mode 100644 index 0000000000..066f8d990d --- /dev/null +++ b/plugins/filter/to_yaml.yml @@ -0,0 +1,92 @@ +# Copyright (c) Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +DOCUMENTATION: + name: to_yaml + author: + - Ansible Core Team + - Felix Fontein (@felixfontein) + version_added: 11.3.0 + short_description: Convert variable to YAML string + description: + - Converts an Ansible variable into a YAML string representation, without preserving vaulted strings as P(ansible.builtin.to_yaml#filter). + - This filter functions as a wrapper to the L(Python PyYAML library, https://pypi.org/project/PyYAML/)'s C(yaml.dump) function. + positional: _input + options: + _input: + description: + - A variable or expression that returns a data structure. + type: raw + required: true + indent: + description: + - Number of spaces to indent Python structures, mainly used for display to humans. + type: integer + sort_keys: + description: + - Affects sorting of dictionary keys. + default: true + type: bool + default_style: + description: + - Indicates the style of the scalar. + choices: + - '' + - "'" + - '"' + - '|' + - '>' + type: string + canonical: + description: + - If set to V(true), export tag type to the output. + type: bool + width: + description: + - Set the preferred line width. + type: integer + line_break: + description: + - Specify the line break. + type: string + encoding: + description: + - Specify the output encoding. + type: string + explicit_start: + description: + - If set to V(true), adds an explicit start using C(---). + type: bool + explicit_end: + description: + - If set to V(true), adds an explicit end using C(...). + type: bool + redact_sensitive_values: + description: + - If set to V(true), vaulted strings are replaced by V() instead of being decrypted. + - With future ansible-core versions, this can extend to other strings tagged as sensitive. + - B(Note) that with ansible-core 2.18 and before this might not yield the expected result + since these versions of ansible-core strip the vault information away from strings that are + part of more complex data structures specified in C(vars). + type: bool + default: false + notes: + - More options may be available, see L(PyYAML documentation, https://pyyaml.org/wiki/PyYAMLDocumentation) for details. + - >- + These parameters to C(yaml.dump) are not accepted, as they are overridden internally: O(ignore:allow_unicode). + +EXAMPLES: | + --- + # Dump variable in a template to create a YAML document + value: "{{ github_workflow | community.general.to_yaml }}" + + --- + # Same as above but 'prettier' (equivalent to community.general.to_nice_yaml filter) + value: "{{ docker_config | community.general.to_yaml(indent=2) }}" + +RETURN: + _value: + description: + - The YAML serialized string representing the variable structure inputted. + type: string diff --git a/tests/integration/targets/filter_to_yaml/aliases b/tests/integration/targets/filter_to_yaml/aliases new file mode 100644 index 0000000000..343f119da8 --- /dev/null +++ b/tests/integration/targets/filter_to_yaml/aliases @@ -0,0 +1,5 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +azp/posix/3 diff --git a/tests/integration/targets/filter_to_yaml/main.yml b/tests/integration/targets/filter_to_yaml/main.yml new file mode 100644 index 0000000000..450430416e --- /dev/null +++ b/tests/integration/targets/filter_to_yaml/main.yml @@ -0,0 +1,188 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- hosts: localhost + gather_facts: false + vars_files: + - vaulted_vars.yml + vars: + timestamp: 2025-01-02T03:04:05Z + bar: foobarbaz + tasks: + - name: Print vaulted values + debug: + msg: + foo: "{{ foo }}" + + - name: Convert values to YAML + set_fact: + vstr: "{{ 'foo' | community.general.to_yaml }}" + vvstr: "{{ foo | community.general.to_yaml }}" + vvstr_redact: "{{ foo | community.general.to_yaml(redact_sensitive_values=true) }}" + vint: "{{ 42 | community.general.to_yaml }}" + vfloat: "{{ -3.1415 | community.general.to_yaml }}" + vbool: "{{ true | community.general.to_yaml }}" + vtimestamp: "{{ timestamp | community.general.to_yaml }}" + vlist: "{{ [1, false, 'bar'] | community.general.to_yaml }}" + vdict: "{{ {'a': 'b', 1: 2} | community.general.to_yaml }}" + + - name: Check values + assert: + that: + - 'vstr == "foo\n"' + - 'vvstr == "bar\n"' + - 'vvstr_redact == "\n"' + - 'vint == "42\n"' + - 'vfloat == "-3.1415\n"' + - 'vbool == "true\n"' + - 'vtimestamp == "2025-01-02 03:04:05+00:00\n"' + - 'vlist == "[1, false, bar]\n"' + - 'vdict == "{a: b, 1: 2}\n"' + + - name: Convert values to nice YAML + set_fact: + vstr: "{{ 'foo' | community.general.to_nice_yaml }}" + vvstr: "{{ foo | community.general.to_nice_yaml }}" + vvstr_redact: "{{ foo | community.general.to_nice_yaml(redact_sensitive_values=true) }}" + vint: "{{ 42 | community.general.to_nice_yaml }}" + vfloat: "{{ -3.1415 | community.general.to_nice_yaml }}" + vbool: "{{ true | community.general.to_nice_yaml }}" + vtimestamp: "{{ timestamp | community.general.to_nice_yaml }}" + vlist: "{{ [1, false, 'bar'] | community.general.to_nice_yaml }}" + vdict: "{{ {'a': 'b', 1: 2} | community.general.to_nice_yaml }}" + + - name: Check values + assert: + that: + - 'vstr == "foo\n"' + - 'vvstr == "bar\n"' + - 'vvstr_redact == "\n"' + - 'vint == "42\n"' + - 'vfloat == "-3.1415\n"' + - 'vbool == "true\n"' + - 'vtimestamp == "2025-01-02 03:04:05+00:00\n"' + - 'vlist == "- 1\n- false\n- bar\n"' + - 'vdict == "a: b\n1: 2\n"' + + - name: Convert more complex data structure (from vars file) + set_fact: + complex: "{{ foobar | community.general.to_yaml }}" + complex_redact: "{{ foobar | community.general.to_yaml(redact_sensitive_values=true) }}" + complex_nice: "{{ foobar | community.general.to_nice_yaml }}" + complex_nice_redact: "{{ foobar | community.general.to_nice_yaml(redact_sensitive_values=true) }}" + + - assert: + that: + - complex == exp_complex + - complex_redact == exp_complex_redact + - complex_nice == exp_complex_nice + - complex_nice_redact == exp_complex_nice_redact + vars: + exp_complex: | + a_list: [bar, ! '2025-02-03 04:05:06', Hello!, true, false] + a_value: 123 + exp_complex_redact: | + a_list: [, ! '2025-02-03 04:05:06', Hello!, true, false] + a_value: 123 + exp_complex_nice: | + a_list: + - bar + - 2025-02-03 04:05:06 + - Hello! + - true + - false + a_value: 123 + exp_complex_nice_redact: | + a_list: + - + - 2025-02-03 04:05:06 + - Hello! + - true + - false + a_value: 123 + + - name: Convert more complex data structure (from vars) + set_fact: + complex: "{{ data | community.general.to_yaml }}" + complex_redact: "{{ data | community.general.to_yaml(redact_sensitive_values=true) }}" + complex_nice: "{{ data | community.general.to_nice_yaml }}" + complex_nice_redact: "{{ data | community.general.to_nice_yaml(redact_sensitive_values=true) }}" + vars: + data: + foo: 123 + bar: 1.23 + baz: true + bam: foobar + bang: + - "{{ timestamp }}" + - "{{ bar }}" + - "{{ foo }}" + + - when: ansible_version.full is version("2.19", "<") + assert: + that: + - complex == exp_complex + # With ansible-core 2.18 and before, the vaulted string is decryped before it reaches the filter, + # so the redaction does not work there. + - complex_redact == exp_complex + - complex_nice == exp_complex_nice + # With ansible-core 2.18 and before, the vaulted string is decryped before it reaches the filter, + # so the redaction does not work there. + - complex_nice_redact == exp_complex_nice + vars: + exp_complex: | + bam: foobar + bang: ['2025-01-02 03:04:05+00:00', foobarbaz, bar] + bar: 1.23 + baz: true + foo: 123 + exp_complex_nice: | + bam: foobar + bang: + - '2025-01-02 03:04:05+00:00' + - foobarbaz + - bar + bar: 1.23 + baz: true + foo: 123 + + - when: ansible_version.full is version("2.19", ">=") + assert: + that: + - complex == exp_complex + - complex_redact == exp_complex_redact + - complex_nice == exp_complex_nice + - complex_nice_redact == exp_complex_nice_redact + vars: + exp_complex: | + bam: foobar + bang: [! '2025-01-02 03:04:05+00:00', foobarbaz, bar] + bar: 1.23 + baz: true + foo: 123 + exp_complex_redact: | + bam: foobar + bang: [! '2025-01-02 03:04:05+00:00', foobarbaz, ] + bar: 1.23 + baz: true + foo: 123 + exp_complex_nice: | + bam: foobar + bang: + - 2025-01-02 03:04:05+00:00 + - foobarbaz + - bar + bar: 1.23 + baz: true + foo: 123 + exp_complex_nice_redact: | + bam: foobar + bang: + - 2025-01-02 03:04:05+00:00 + - foobarbaz + - + bar: 1.23 + baz: true + foo: 123 diff --git a/tests/integration/targets/filter_to_yaml/password b/tests/integration/targets/filter_to_yaml/password new file mode 100644 index 0000000000..d97c5eada5 --- /dev/null +++ b/tests/integration/targets/filter_to_yaml/password @@ -0,0 +1 @@ +secret diff --git a/tests/integration/targets/filter_to_yaml/password.license b/tests/integration/targets/filter_to_yaml/password.license new file mode 100644 index 0000000000..a1390a69ed --- /dev/null +++ b/tests/integration/targets/filter_to_yaml/password.license @@ -0,0 +1,3 @@ +Copyright (c) Ansible Project +GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +SPDX-License-Identifier: GPL-3.0-or-later diff --git a/tests/integration/targets/filter_to_yaml/runme.sh b/tests/integration/targets/filter_to_yaml/runme.sh new file mode 100755 index 0000000000..a91c72b563 --- /dev/null +++ b/tests/integration/targets/filter_to_yaml/runme.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +set -eux + +ansible-playbook --vault-password-file password main.yml "$@" diff --git a/tests/integration/targets/filter_to_yaml/vaulted_vars.yml b/tests/integration/targets/filter_to_yaml/vaulted_vars.yml new file mode 100644 index 0000000000..c8f7c6141d --- /dev/null +++ b/tests/integration/targets/filter_to_yaml/vaulted_vars.yml @@ -0,0 +1,27 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +foo: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 32336431346561346535396563363438333131636539653331376466383331663838303835353862 + 3536306130663166393533626530646435383938323066320a303366613035323835373030303262 + 35633636653362393531653961396665663965356562346538643863336562393734376234313134 + 3562663234326435390a376464633234373636643538353562326133316439343863373333363265 + 6239 + +foobar: + a_value: 123 + a_list: + - !vault | + $ANSIBLE_VAULT;1.1;AES256 + 32336431346561346535396563363438333131636539653331376466383331663838303835353862 + 3536306130663166393533626530646435383938323066320a303366613035323835373030303262 + 35633636653362393531653961396665663965356562346538643863336562393734376234313134 + 3562663234326435390a376464633234373636643538353562326133316439343863373333363265 + 6239 + - 2025-02-03 04:05:06 + - Hello! + - true + - false