[PR #7985/102a0857 backport][stable-8] New filters to calculate the union, intersection, difference and symmetric difference of lists by preserving the items order (#8020)

New filters to calculate the union, intersection, difference and symmetric difference of lists by preserving the items order (#7985)

New filters lists_union, lists_intersect, lists_difference and lists_symmetric_difference added.

Signed-off-by: Christoph Fiehe <c.fiehe@eurodata.de>
Co-authored-by: Christoph Fiehe <c.fiehe@eurodata.de>
(cherry picked from commit 102a0857db)

Co-authored-by: cfiehe <cfiehe@users.noreply.github.com>

plugins/filter/lists.py

@@ -0,0 +1,210 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from ansible.errors import AnsibleFilterError
+from ansible.module_utils.common.collections import is_sequence
+
+
+def remove_duplicates(lst):
+    seen = set()
+    seen_add = seen.add
+    result = []
+    for item in lst:
+        try:
+            if item not in seen:
+                seen_add(item)
+                result.append(item)
+        except TypeError:
+            # This happens for unhashable values `item`. If this happens,
+            # convert `seen` to a list and continue.
+            seen = list(seen)
+            seen_add = seen.append
+            if item not in seen:
+                seen_add(item)
+                result.append(item)
+    return result
+
+
+def flatten_list(lst):
+    result = []
+    for sublist in lst:
+        if not is_sequence(sublist):
+            msg = ("All arguments must be lists. %s is %s")
+            raise AnsibleFilterError(msg % (sublist, type(sublist)))
+        if len(sublist) > 0:
+            if all(is_sequence(sub) for sub in sublist):
+                for item in sublist:
+                    result.append(item)
+            else:
+                result.append(sublist)
+    return result
+
+
+def lists_union(*args, **kwargs):
+    lists = args
+    flatten = kwargs.pop('flatten', False)
+
+    if kwargs:
+        # Some unused kwargs remain
+        raise AnsibleFilterError(
+            "lists_union() got unexpected keywords arguments: {0}".format(
+                ", ".join(kwargs.keys())
+            )
+        )
+
+    if flatten:
+        lists = flatten_list(args)
+
+    if not lists:
+        return []
+
+    if len(lists) == 1:
+        return lists[0]
+
+    a = lists[0]
+    for b in lists[1:]:
+        a = do_union(a, b)
+    return remove_duplicates(a)
+
+
+def do_union(a, b):
+    return a + b
+
+
+def lists_intersect(*args, **kwargs):
+    lists = args
+    flatten = kwargs.pop('flatten', False)
+
+    if kwargs:
+        # Some unused kwargs remain
+        raise AnsibleFilterError(
+            "lists_intersect() got unexpected keywords arguments: {0}".format(
+                ", ".join(kwargs.keys())
+            )
+        )
+
+    if flatten:
+        lists = flatten_list(args)
+
+    if not lists:
+        return []
+
+    if len(lists) == 1:
+        return lists[0]
+
+    a = remove_duplicates(lists[0])
+    for b in lists[1:]:
+        a = do_intersect(a, b)
+    return a
+
+
+def do_intersect(a, b):
+    isect = []
+    try:
+        other = set(b)
+        isect = [item for item in a if item in other]
+    except TypeError:
+        # This happens for unhashable values,
+        # use a list instead and redo.
+        other = list(b)
+        isect = [item for item in a if item in other]
+    return isect
+
+
+def lists_difference(*args, **kwargs):
+    lists = args
+    flatten = kwargs.pop('flatten', False)
+
+    if kwargs:
+        # Some unused kwargs remain
+        raise AnsibleFilterError(
+            "lists_difference() got unexpected keywords arguments: {0}".format(
+                ", ".join(kwargs.keys())
+            )
+        )
+
+    if flatten:
+        lists = flatten_list(args)
+
+    if not lists:
+        return []
+
+    if len(lists) == 1:
+        return lists[0]
+
+    a = remove_duplicates(lists[0])
+    for b in lists[1:]:
+        a = do_difference(a, b)
+    return a
+
+
+def do_difference(a, b):
+    diff = []
+    try:
+        other = set(b)
+        diff = [item for item in a if item not in other]
+    except TypeError:
+        # This happens for unhashable values,
+        # use a list instead and redo.
+        other = list(b)
+        diff = [item for item in a if item not in other]
+    return diff
+
+
+def lists_symmetric_difference(*args, **kwargs):
+    lists = args
+    flatten = kwargs.pop('flatten', False)
+
+    if kwargs:
+        # Some unused kwargs remain
+        raise AnsibleFilterError(
+            "lists_difference() got unexpected keywords arguments: {0}".format(
+                ", ".join(kwargs.keys())
+            )
+        )
+
+    if flatten:
+        lists = flatten_list(args)
+
+    if not lists:
+        return []
+
+    if len(lists) == 1:
+        return lists[0]
+
+    a = lists[0]
+    for b in lists[1:]:
+        a = do_symmetric_difference(a, b)
+    return a
+
+
+def do_symmetric_difference(a, b):
+    sym_diff = []
+    union = lists_union(a, b)
+    try:
+        isect = set(a) & set(b)
+        sym_diff = [item for item in union if item not in isect]
+    except TypeError:
+        # This happens for unhashable values,
+        # build the intersection of `a` and `b` backed
+        # by a list instead of a set and redo.
+        isect = lists_intersect(a, b)
+        sym_diff = [item for item in union if item not in isect]
+    return sym_diff
+
+
+class FilterModule(object):
+    ''' Ansible lists jinja2 filters '''
+
+    def filters(self):
+        return {
+            'lists_union': lists_union,
+            'lists_intersect': lists_intersect,
+            'lists_difference': lists_difference,
+            'lists_symmetric_difference': lists_symmetric_difference,
+        }

plugins/filter/lists_difference.yml

@@ -0,0 +1,48 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+DOCUMENTATION:
+  name: lists_difference
+  short_description: Difference of lists with a predictive order
+  version_added: 8.4.0
+  description:
+    - Provide a unique list of all the elements from the first which do not appear in the other lists.
+    - The order of the items in the resulting list is preserved.
+  options:
+    _input:
+      description: A list.
+      type: list
+      elements: any
+      required: true
+    flatten:
+      description: Whether to remove one hierarchy level from the input list.
+      type: boolean
+      default: false
+  author:
+    - Christoph Fiehe (@cfiehe)
+
+EXAMPLES: |
+  - name: Return the difference of list1 and list2.
+    ansible.builtin.debug:
+      msg: "{{ list1 | community.general.lists_difference(list2) }}"
+    vars:
+      list1: [1, 2, 5, 3, 4, 10]
+      list2: [1, 2, 3, 4, 5, 11, 99]
+  # => [10]
+  
+  - name: Return the difference of list1, list2 and list3.
+    ansible.builtin.debug:
+      msg: "{{ [list1, list2, list3] | community.general.lists_difference(flatten=true) }}"
+    vars:
+      list1: [1, 2, 5, 3, 4, 10]
+      list2: [1, 2, 3, 4, 5, 11, 99]
+      list3: [1, 2, 3, 4, 5, 10, 99, 101]
+  # => []
+
+RETURN:
+  _value:
+    description: A unique list of all the elements from the first list that do not appear on the other lists.
+    type: list
+    elements: any

plugins/filter/lists_intersect.yml

@@ -0,0 +1,48 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+DOCUMENTATION:
+  name: lists_intersect
+  short_description: Intersection of lists with a predictive order
+  version_added: 8.4.0
+  description:
+    - Provide a unique list of all the common elements of two or more lists.
+    - The order of the items in the resulting list is preserved.
+  options:
+    _input:
+      description: A list.
+      type: list
+      elements: any
+      required: true
+    flatten:
+      description: Whether to remove one hierarchy level from the input list.
+      type: boolean
+      default: false
+  author:
+    - Christoph Fiehe (@cfiehe)
+
+EXAMPLES: |
+  - name: Return the intersection of list1 and list2.
+    ansible.builtin.debug:
+      msg: "{{ list1 | community.general.lists_intersect(list2) }}"
+    vars:
+      list1: [1, 2, 5, 3, 4, 10]
+      list2: [1, 2, 3, 4, 5, 11, 99]
+  # => [1, 2, 5, 3, 4]
+  
+  - name: Return the intersection of list1, list2 and list3.
+    ansible.builtin.debug:
+      msg: "{{ [list1, list2, list3] | community.general.lists_intersect(flatten=true) }}"
+    vars:
+      list1: [1, 2, 5, 3, 4, 10]
+      list2: [1, 2, 3, 4, 5, 11, 99]
+      list3: [1, 2, 3, 4, 5, 10, 99, 101]
+  # => [1, 2, 5, 3, 4]
+
+RETURN:
+  _value:
+    description: A unique list of all the common elements from the provided lists.
+    type: list
+    elements: any

plugins/filter/lists_symmetric_difference.yml

@@ -0,0 +1,48 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+DOCUMENTATION:
+  name: lists_symmetric_difference
+  short_description: Symmetric Difference of lists with a predictive order
+  version_added: 8.4.0
+  description:
+    - Provide a unique list containing the symmetric difference of two or more lists.
+    - The order of the items in the resulting list is preserved.
+  options:
+    _input:
+      description: A list.
+      type: list
+      elements: any
+      required: true
+    flatten:
+      description: Whether to remove one hierarchy level from the input list.
+      type: boolean
+      default: false
+  author:
+    - Christoph Fiehe (@cfiehe)
+
+EXAMPLES: |
+  - name: Return the symmetric difference of list1 and list2.
+    ansible.builtin.debug:
+      msg: "{{ list1 | community.general.lists_symmetric_difference(list2) }}"
+    vars:
+      list1: [1, 2, 5, 3, 4, 10]
+      list2: [1, 2, 3, 4, 5, 11, 99]
+  # => [10, 11, 99]
+  
+  - name: Return the symmetric difference of list1, list2 and list3.
+    ansible.builtin.debug:
+      msg: "{{ [list1, list2, list3] | community.general.lists_symmetric_difference(flatten=true) }}"
+    vars:
+      list1: [1, 2, 5, 3, 4, 10]
+      list2: [1, 2, 3, 4, 5, 11, 99]
+      list3: [1, 2, 3, 4, 5, 10, 99, 101]
+  # => [11, 1, 2, 3, 4, 5, 101]
+
+RETURN:
+  _value:
+    description: A unique list containing the symmetric difference of two or more lists.
+    type: list
+    elements: any

plugins/filter/lists_union.yml

@@ -0,0 +1,48 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+DOCUMENTATION:
+  name: lists_union
+  short_description: Union of lists with a predictive order
+  version_added: 8.4.0
+  description:
+    - Provide a unique list of all the elements of two or more lists.
+    - The order of the items in the resulting list is preserved.
+  options:
+    _input:
+      description: A list.
+      type: list
+      elements: any
+      required: true
+    flatten:
+      description: Whether to remove one hierarchy level from the input list.
+      type: boolean
+      default: false
+  author:
+    - Christoph Fiehe (@cfiehe)
+
+EXAMPLES: |
+  - name: Return the union of list1, list2 and list3.
+    ansible.builtin.debug:
+      msg: "{{ list1 | community.general.lists_union(list2, list3) }}"
+    vars:
+      list1: [1, 2, 5, 3, 4, 10]
+      list2: [1, 2, 3, 4, 5, 11, 99]
+      list3: [1, 2, 3, 4, 5, 10, 99, 101]
+  # => [1, 2, 5, 3, 4, 10, 11, 99, 101]
+  
+  - name: Return the union of list1 and list2.
+    ansible.builtin.debug:
+      msg: "{{ [list1, list2] | community.general.lists_union(flatten=true) }}"
+    vars:
+      list1: [1, 2, 5, 3, 4, 10]
+      list2: [1, 2, 3, 4, 5, 11, 99]
+  # => [1, 2, 5, 3, 4, 10, 11, 99]
+
+RETURN:
+  _value:
+    description: A unique list of all the elements from the provided lists.
+    type: list
+    elements: any