From f18cb9d8b64e29c564de565d7ef4dc6c241110c5 Mon Sep 17 00:00:00 2001 From: Lorenzo Tanganelli Date: Wed, 14 May 2025 21:14:13 +0200 Subject: [PATCH 1/5] add remove keys from values --- plugins/filter/remove_keys_from_values.py | 217 ++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 plugins/filter/remove_keys_from_values.py diff --git a/plugins/filter/remove_keys_from_values.py b/plugins/filter/remove_keys_from_values.py new file mode 100644 index 0000000000..27e80d22d5 --- /dev/null +++ b/plugins/filter/remove_keys_from_values.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2024 Vladimir Botka +# Copyright (c) 2024 Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import annotations + +DOCUMENTATION = r""" +name: remove_keys_from_values +short_description: Remove keys from a list of dictionaries or a dictionary in base of value content +version_added: "10.6.0" +author: + - Lorenzo Tanganelli (@tanganellilore) +description: + - This filter removes keys from a list of dictionaries or a dictionary, + - recursively or not depending on the parameters. + - The type of values to be removed is defined by the O(values) parameter. + - The default is to remove keys with values like C(''), C([]), C({}), C(None), usefull for removing empty keys. +options: + _input: + description: + - A list of dictionaries or a dictionary. + type: raw + elements: dictionary + required: true + values: + description: + - A single value or value pattern to remove, or a list of values or values patterns to remove. + - If O(matching_parameter=regex) there must be equally one pattern provided. + type: raw + required: true + recursive: + description: Specify if the filter should be applied recursively. + type: bool + default: true + required: false + matching_parameter: + description: Specify the matching option of target keys. + type: str + default: equal + choices: + equal: Matches keys of equally one of the O(target[].before) items. + regex: Matches keys that match one of the regular expressions provided in O(target[].before).: +""" + +EXAMPLES = r""" +- name: Remove empty keys from a list of dictionaries + set_fact: + my_list: + - a: foo + b: '' + c: [] + - a: bar + b: {} + c: None + - a: ok + b: {} + c: None + - debug: + msg: "{{ my_list | remove_empty_keys }}" + - debug: + msg: "{{ my_list | remove_empty_keys(values='') }}" + - debug: + msg: "{{ my_list | remove_empty_keys(values=['', [], {}, None]) }}" + - debug: + msg: "{{ my_list | remove_empty_keys(values=['', [], {}, None], recursive=False) }}" + - debug: + msg: "{{ my_list | remove_empty_keys(values=['foo', 'bar']) }}" + +- name: Remove keys from a dictionary + set_fact: + my_dict: + a: foo + b: '' + c: [] + d: + - a: foo + b: '' + c: [] + - a: bar + b: {} + c: None + - debug: + msg: "{{ my_dict | remove_empty_keys }}" + # returns + # a: foo + # d: + # - a: foo + # - a: bar + - debug: + msg: "{{ my_dict | remove_empty_keys(values='') }}" + # returns + # a: foo + # d: + # - a: foo + # c: [] + # - a: bar + # b: {} + # c: None + + - debug: + msg: "{{ my_dict | remove_empty_keys(values=['', [], {}, None], recursive=False) }}" + # return + # a: foo + # d: + # - a: foo + # - a: bar + + - debug: + msg: "{{ my_dict | remove_empty_keys(values=['foo', 'bar']) }}" + # returns + # b: '' + # c: [] + # d: + # - b: '' + # c: [] + # - b: {} + # c: None +""" + +RETURN = r""" +_value: + description: The list of dictionaries or the dictionary with the keys removed. + returned: always + type: raw +""" + +from ansible.errors import AnsibleFilterError + + +def remove_keys_from_value(data, values=None, recursive=True): + """ + Removes keys from dictionaries or lists of dictionaries + if their value matches any of the specified `values`. + """ + # Default values to remove + default_values = ['', [], {}, None] + if values is None: + values = default_values + elif not isinstance(values, list): + values = [values] + + def should_remove(val): + return val in values + + def clean(obj): + if isinstance(obj, dict): + new_obj = {} + for k, v in obj.items(): + val = clean(v) if recursive else v + if not should_remove(val): + new_obj[k] = val + return new_obj + elif isinstance(obj, list): + return [clean(item) if recursive else item for item in obj] + else: + return obj + + if isinstance(data, (dict, list)): + return clean(data) + else: + raise AnsibleFilterError("Input must be a dictionary or list of dictionaries.") + + +def remove_keys_from_values(data, values=None, recursive=True, matching_parameter="equal"): + """ + Removes keys from dictionaries or lists of dictionaries + if their values match the specified values or regex patterns. + """ + default_values = ['', [], {}, None] + if values is None: + values = default_values + elif not isinstance(values, list): + values = [values] + + if matching_parameter not in ["equal", "regex"]: + raise AnsibleFilterError(f"Invalid matching_parameter '{matching_parameter}'. Use 'equal' or 'regex'.") + + regex_patterns = [] + if matching_parameter == "regex": + # Convert string patterns to compiled regex + try: + regex_patterns = [re.compile(v) for v in values] + except re.error as e: + raise AnsibleFilterError(f"Invalid regex pattern: {e}") + + def should_remove(val): + if matching_parameter == "equal": + return val in values + elif matching_parameter == "regex" and isinstance(val, str): + return any(p.match(val) for p in regex_patterns) + return False + + def clean(obj): + if isinstance(obj, dict): + return { + k: clean(v) if recursive else v + for k, v in obj.items() + if not should_remove(clean(v) if recursive else v) + } + elif isinstance(obj, list): + return [clean(item) if recursive else item for item in obj] + else: + return obj + + if isinstance(data, (dict, list)): + return clean(data) + else: + raise AnsibleFilterError("Input must be a dictionary or a list of dictionaries.") + + +class FilterModule(object): + def filters(self): + return { + 'remove_keys_from_values': remove_keys_from_values + } From 9f02444e55b3c3b4ad51352981442362f85abec8 Mon Sep 17 00:00:00 2001 From: Lorenzo Tanganelli Date: Thu, 15 May 2025 17:11:39 +0200 Subject: [PATCH 2/5] optimize and import re --- plugins/filter/remove_keys_from_values.py | 29 ++++++++++------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/plugins/filter/remove_keys_from_values.py b/plugins/filter/remove_keys_from_values.py index 27e80d22d5..786a4d2239 100644 --- a/plugins/filter/remove_keys_from_values.py +++ b/plugins/filter/remove_keys_from_values.py @@ -126,6 +126,7 @@ _value: type: raw """ +import re from ansible.errors import AnsibleFilterError @@ -168,18 +169,17 @@ def remove_keys_from_values(data, values=None, recursive=True, matching_paramete Removes keys from dictionaries or lists of dictionaries if their values match the specified values or regex patterns. """ - default_values = ['', [], {}, None] - if values is None: - values = default_values - elif not isinstance(values, list): - values = [values] + + if not isinstance(data, (dict, list)): + raise AnsibleFilterError("Input must be a dictionary or a list.") - if matching_parameter not in ["equal", "regex"]: - raise AnsibleFilterError(f"Invalid matching_parameter '{matching_parameter}'. Use 'equal' or 'regex'.") + if matching_parameter not in ("equal", "regex"): + raise AnsibleFilterError("matching_parameter must be 'equal' or 'regex'") + + values = values if isinstance(values, list) else [values or '', [], {}, None] regex_patterns = [] if matching_parameter == "regex": - # Convert string patterns to compiled regex try: regex_patterns = [re.compile(v) for v in values] except re.error as e: @@ -188,7 +188,7 @@ def remove_keys_from_values(data, values=None, recursive=True, matching_paramete def should_remove(val): if matching_parameter == "equal": return val in values - elif matching_parameter == "regex" and isinstance(val, str): + if matching_parameter == "regex" and isinstance(val, str): return any(p.match(val) for p in regex_patterns) return False @@ -200,15 +200,10 @@ def remove_keys_from_values(data, values=None, recursive=True, matching_paramete if not should_remove(clean(v) if recursive else v) } elif isinstance(obj, list): - return [clean(item) if recursive else item for item in obj] - else: - return obj - - if isinstance(data, (dict, list)): - return clean(data) - else: - raise AnsibleFilterError("Input must be a dictionary or a list of dictionaries.") + return [clean(i) if recursive else i for i in obj] + return obj + return clean(data) class FilterModule(object): def filters(self): From 2622de7fef15ff27e5819a3698e0b5d36a0172cf Mon Sep 17 00:00:00 2001 From: Lorenzo Tanganelli Date: Thu, 15 May 2025 17:21:08 +0200 Subject: [PATCH 3/5] add changelog --- changelogs/fragments/10139-add-remove-keys-from-values.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/10139-add-remove-keys-from-values.yml diff --git a/changelogs/fragments/10139-add-remove-keys-from-values.yml b/changelogs/fragments/10139-add-remove-keys-from-values.yml new file mode 100644 index 0000000000..7d8480f202 --- /dev/null +++ b/changelogs/fragments/10139-add-remove-keys-from-values.yml @@ -0,0 +1,2 @@ +minor_changes: + - remove_keys_from_values - add new filter plugin \ No newline at end of file From 7bf066116b4a6f6022ac48878a64db766cf7f207 Mon Sep 17 00:00:00 2001 From: Lorenzo Tanganelli Date: Thu, 15 May 2025 15:39:27 +0000 Subject: [PATCH 4/5] fix lint --- ansible | 1 + plugins/filter/remove_keys_from_values.py | 99 ++++++++++++----------- 2 files changed, 51 insertions(+), 49 deletions(-) create mode 160000 ansible diff --git a/ansible b/ansible new file mode 160000 index 0000000000..fe2d9e316a --- /dev/null +++ b/ansible @@ -0,0 +1 @@ +Subproject commit fe2d9e316a38c96b798e830acc0a1314c35b8ef4 diff --git a/plugins/filter/remove_keys_from_values.py b/plugins/filter/remove_keys_from_values.py index 786a4d2239..0a2d3cb445 100644 --- a/plugins/filter/remove_keys_from_values.py +++ b/plugins/filter/remove_keys_from_values.py @@ -36,16 +36,16 @@ options: default: true required: false matching_parameter: - description: Specify the matching option of target keys. + description: Specify the matching option of values. type: str default: equal choices: - equal: Matches keys of equally one of the O(target[].before) items. - regex: Matches keys that match one of the regular expressions provided in O(target[].before).: + equal: Matches values of equally one of the O(values[]) items. + regex: Matches values that match one of the regular expressions provided in O(values[]). """ EXAMPLES = r""" -- name: Remove empty keys from a list of dictionaries +- name: Remove empty values from a list of dictionaries set_fact: my_list: - a: foo @@ -57,16 +57,16 @@ EXAMPLES = r""" - a: ok b: {} c: None - - debug: - msg: "{{ my_list | remove_empty_keys }}" - - debug: - msg: "{{ my_list | remove_empty_keys(values='') }}" - - debug: - msg: "{{ my_list | remove_empty_keys(values=['', [], {}, None]) }}" - - debug: - msg: "{{ my_list | remove_empty_keys(values=['', [], {}, None], recursive=False) }}" - - debug: - msg: "{{ my_list | remove_empty_keys(values=['foo', 'bar']) }}" +- debug: + msg: "{{ my_list | remove_empty_keys }}" +- debug: + msg: "{{ my_list | remove_empty_keys(values='') }}" +- debug: + msg: "{{ my_list | remove_empty_keys(values=['', [], {}, None]) }}" +- debug: + msg: "{{ my_list | remove_empty_keys(values=['', [], {}, None], recursive=False) }}" +- debug: + msg: "{{ my_list | remove_empty_keys(values=['foo', 'bar']) }}" - name: Remove keys from a dictionary set_fact: @@ -81,42 +81,42 @@ EXAMPLES = r""" - a: bar b: {} c: None - - debug: - msg: "{{ my_dict | remove_empty_keys }}" - # returns - # a: foo - # d: - # - a: foo - # - a: bar - - debug: - msg: "{{ my_dict | remove_empty_keys(values='') }}" - # returns - # a: foo - # d: - # - a: foo - # c: [] - # - a: bar - # b: {} - # c: None +- debug: + msg: "{{ my_dict | remove_empty_keys }}" +# returns +# a: foo +# d: +# - a: foo +# - a: bar +- debug: + msg: "{{ my_dict | remove_empty_keys(values='') }}" +# returns +# a: foo +# d: +# - a: foo +# c: [] +# - a: bar +# b: {} +# c: None - - debug: - msg: "{{ my_dict | remove_empty_keys(values=['', [], {}, None], recursive=False) }}" - # return - # a: foo - # d: - # - a: foo - # - a: bar +- debug: + msg: "{{ my_dict | remove_empty_keys(values=['', [], {}, None], recursive=False) }}" +# return +# a: foo +# d: +# - a: foo +# - a: bar - - debug: - msg: "{{ my_dict | remove_empty_keys(values=['foo', 'bar']) }}" - # returns - # b: '' - # c: [] - # d: - # - b: '' - # c: [] - # - b: {} - # c: None +- debug: + msg: "{{ my_dict | remove_empty_keys(values=['foo', 'bar']) }}" +# returns +# b: '' +# c: [] +# d: +# - b: '' +# c: [] +# - b: {} +# c: None """ RETURN = r""" @@ -169,7 +169,7 @@ def remove_keys_from_values(data, values=None, recursive=True, matching_paramete Removes keys from dictionaries or lists of dictionaries if their values match the specified values or regex patterns. """ - + if not isinstance(data, (dict, list)): raise AnsibleFilterError("Input must be a dictionary or a list.") @@ -205,6 +205,7 @@ def remove_keys_from_values(data, values=None, recursive=True, matching_paramete return clean(data) + class FilterModule(object): def filters(self): return { From 0d976f71ad5d28ca198935c67dc9a68f9df8ad7b Mon Sep 17 00:00:00 2001 From: Lorenzo Tanganelli Date: Fri, 30 May 2025 10:51:45 +0000 Subject: [PATCH 5/5] fix after first review --- ansible | 1 - changelogs/fragments/10139-add-remove-keys-from-values.yml | 2 -- plugins/filter/remove_keys_from_values.py | 4 ++-- 3 files changed, 2 insertions(+), 5 deletions(-) delete mode 160000 ansible delete mode 100644 changelogs/fragments/10139-add-remove-keys-from-values.yml diff --git a/ansible b/ansible deleted file mode 160000 index fe2d9e316a..0000000000 --- a/ansible +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fe2d9e316a38c96b798e830acc0a1314c35b8ef4 diff --git a/changelogs/fragments/10139-add-remove-keys-from-values.yml b/changelogs/fragments/10139-add-remove-keys-from-values.yml deleted file mode 100644 index 7d8480f202..0000000000 --- a/changelogs/fragments/10139-add-remove-keys-from-values.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - remove_keys_from_values - add new filter plugin \ No newline at end of file diff --git a/plugins/filter/remove_keys_from_values.py b/plugins/filter/remove_keys_from_values.py index 0a2d3cb445..7d19fa0d09 100644 --- a/plugins/filter/remove_keys_from_values.py +++ b/plugins/filter/remove_keys_from_values.py @@ -9,12 +9,12 @@ from __future__ import annotations DOCUMENTATION = r""" name: remove_keys_from_values short_description: Remove keys from a list of dictionaries or a dictionary in base of value content -version_added: "10.6.0" +version_added: "11.0.0" author: - Lorenzo Tanganelli (@tanganellilore) description: - This filter removes keys from a list of dictionaries or a dictionary, - - recursively or not depending on the parameters. + recursively or not depending on the parameters. - The type of values to be removed is defined by the O(values) parameter. - The default is to remove keys with values like C(''), C([]), C({}), C(None), usefull for removing empty keys. options: