Add new merge_variables lookup plugin (#5533)

* Add new merge_variables lookup plugin

* Add changelog fragment

* Process bot feedback

* Refactor override options and add pattern_type option

* Fix unit tests

* Changed default pattern_type and simplified plugin based on feedback

* Processed feedback for merge_variables lookup plugin

* Adjust version_added.

---------

Co-authored-by: Felix Fontein <felix@fontein.de>

tests/integration/targets/lookup_merge_variables/aliases

@@ -0,0 +1,6 @@
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+azp/posix/1
+skip/python2.6  # lookups are controller only, and we no longer support Python 2.6 on the controller

tests/integration/targets/lookup_merge_variables/runme.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+set -eux
+
+ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
+    ansible-playbook test.yml "$@"
+
+ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
+ANSIBLE_MERGE_VARIABLES_PATTERN_TYPE=suffix \
+    ansible-playbook test_with_env.yml "$@"

tests/integration/targets/lookup_merge_variables/test.yml

@@ -0,0 +1,174 @@
+---
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Test merge_variables lookup plugin
+  hosts: localhost
+  tasks:
+    - name: Include test data
+      include_vars: vars.yml
+
+    # Test the default behavior
+    - name: Test merge list
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 2"
+              - "'item1' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', '^.+__merge_list$') }}"
+
+    - name: Test merge dict
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_dict }}"
+
+        - name: Validate that dict is complete
+          assert:
+            that:
+              - "'item1' in merged_dict"
+              - "'item2' in merged_dict"
+              - "'list_item' in merged_dict"
+              - "(merged_dict.list_item | length) == 2"
+              - "'test1' in (merged_dict.list_item)"
+              - "'test2' in (merged_dict.list_item)"
+      vars:
+        merged_dict: "{{ lookup('community.general.merge_variables', '^.+__merge_dict$') }}"
+
+    # Test the behavior when no results are found
+    - name: Test merge without results
+      block:
+        - debug:
+            msg: "{{ not_found }}"
+        - name: Validate that the variable defaults to an empty list
+          vars:
+          assert:
+            that:
+              - "(not_found | default('default-used', True)) == 'default-used'"
+      vars:
+        not_found: "{{ lookup('community.general.merge_variables', '^.+__merge_not_found$') }}"
+
+    # Test the 'pattern_type' options
+    - name: Test merge list (pattern_type = prefix)
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 4"
+              - "'item1' in merged_list"
+              - "'item2' in merged_list"
+              - "'item2' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', 'testlist', pattern_type='prefix') }}"
+
+    - name: Test merge list (pattern_type = suffix)
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 2"
+              - "'item1' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', '__merge_list', pattern_type='suffix') }}"
+
+    - name: Test merge list (pattern_type = regex)
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 3"
+              - "'item1' in merged_list"
+              - "'item2' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', '^testlist[0-9].*', pattern_type='regex') }}"
+
+    # Test the 'initial_value' option
+    - name: Test merge without results but with initial value
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ not_found_initial_value }}"
+
+        - name: Validate that the variable only contains the initial value
+          vars:
+          assert:
+            that:
+              - "(not_found_initial_value | count) == 1"
+              - "(not_found_initial_value | first) == 'item2'"
+      vars:
+        not_found_initial_value: "{{ lookup('community.general.merge_variables', '^.+__merge_not_found$', initial_value=testlist_initial_value) }}"
+
+    - name: Test merging a list with an initial value
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list_with_initial_value }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list_with_initial_value | length) == 3"
+              - "'item1' in merged_list_with_initial_value"
+              - "'item2' in merged_list_with_initial_value"
+              - "'item3' in merged_list_with_initial_value"
+      vars:
+        merged_list_with_initial_value: "{{ lookup('community.general.merge_variables', '^.+__merge_list$', initial_value=testlist_initial_value) }}"
+
+    # Test the 'override' options
+    - name: Test the 'override=warn' option
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_with_override_warn }}"
+
+        - name: Validate that the dict is complete and the warning is printed
+          assert:
+            that:
+              - "'key_to_override' in merged_with_override_warn"
+              - "merged_with_override_warn.key_to_override == 'Override value'"
+              - "'key_to_override' in lookup('file', logging_output_file)"  # Check if a message is given
+              - "'[WARNING]' in lookup('file', logging_output_file)"        # and verify that the message is a WARNING
+      vars:
+        merged_with_override_warn: "{{ lookup('community.general.merge_variables', '^.+__override_warn$', initial_value=override_warn_init, override='warn') }}"
+
+    - name: Test the 'override=error' option
+      block:
+        - name: Validate that an override result in an error
+          debug:
+            msg: "{{ lookup('community.general.merge_variables', '^.+__override_error$', initial_value=override_error_init, override='error') }}"
+          ignore_errors: true  # Do not stop the playbook
+          register: _override_error_result
+
+        - name: Print the output
+          debug:
+            msg: "{{ _override_error_result }}"
+
+        - name: Validate that the error is reported
+          assert:
+            that:
+              - "_override_error_result.failed"
+              - "'key_to_override' in _override_error_result.msg"

tests/integration/targets/lookup_merge_variables/test_with_env.yml

@@ -0,0 +1,44 @@
+---
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Test merge_variables lookup plugin
+  hosts: localhost
+  tasks:
+    - name: Include test data
+      include_vars: vars.yml
+
+    # Test the pattern option using the environment variable
+    - name: Test merge list (pattern_type = regex)
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 2"
+              - "'item1' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', '__merge_list') }}"
+
+    # Test whether the pattern option can be overridden
+    - name: Test merge list (pattern_type = suffix)
+      block:
+        - name: Print the merged list
+          debug:
+            msg: "{{ merged_list }}"
+
+        - name: Validate that the list is complete
+          assert:
+            that:
+              - "(merged_list | length) == 3"
+              - "'item1' in merged_list"
+              - "'item2' in merged_list"
+              - "'item3' in merged_list"
+      vars:
+        merged_list: "{{ lookup('community.general.merge_variables', '^testlist[0-9].*', pattern_type='regex') }}"

tests/integration/targets/lookup_merge_variables/vars.yml

@@ -0,0 +1,34 @@
+---
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+testlist_initial_value: "{{ testlist2 }}"
+testlist1__merge_list:
+  - item1
+testlist2:
+  - item2
+testlist3__merge_list:
+  - item3
+
+testdict1__merge_dict:
+  item1: test
+  list_item:
+    - test1
+testdict2__merge_dict:
+  item2: test
+  list_item:
+    - test2
+
+override_warn_init:
+  key_to_override: Initial value
+override__override_warn:
+  key_to_override: Override value
+
+override_error_init:
+  key_to_override: Initial value
+override__override_error:
+  key_to_override: Override value
+
+logging_output_file: /tmp/ansible-test-merge-variables  # The Ansible log output is available in this file

tests/unit/plugins/lookup/test_merge_variables.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible_collections.community.general.tests.unit.compat import unittest
+from ansible_collections.community.general.tests.unit.compat.mock import patch
+from ansible_collections.community.general.tests.unit.mock.loader import DictDataLoader
+
+from ansible.plugins import AnsiblePlugin
+from ansible.template import Templar
+from ansible.errors import AnsibleError
+from ansible.utils.display import Display
+from ansible_collections.community.general.plugins.lookup import merge_variables
+
+
+class TestMergeVariablesLookup(unittest.TestCase):
+    def setUp(self):
+        self.loader = DictDataLoader({})
+        self.templar = Templar(loader=self.loader, variables={})
+        self.merge_vars_lookup = merge_variables.LookupModule(loader=self.loader, templar=self.templar)
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[['item1'], ['item3']])
+    def test_merge_list(self, mock_set_options, mock_get_option, mock_template):
+        results = self.merge_vars_lookup.run(['__merge_list'], {
+            'testlist1__merge_list': ['item1'],
+            'testlist2': ['item2'],
+            'testlist3__merge_list': ['item3']
+        })
+
+        self.assertEqual(results, [['item1', 'item3']])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[['initial_item'], 'ignore', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[['item1'], ['item3']])
+    def test_merge_list_with_initial_value(self, mock_set_options, mock_get_option, mock_template):
+        results = self.merge_vars_lookup.run(['__merge_list'], {
+            'testlist1__merge_list': ['item1'],
+            'testlist2': ['item2'],
+            'testlist3__merge_list': ['item3']
+        })
+
+        self.assertEqual(results, [['initial_item', 'item1', 'item3']])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
+                                                    {'item2': 'test', 'list_item': ['test2']}])
+    def test_merge_dict(self, mock_set_options, mock_get_option, mock_template):
+        results = self.merge_vars_lookup.run(['__merge_dict'], {
+            'testdict1__merge_dict': {
+                'item1': 'test',
+                'list_item': ['test1']
+            },
+            'testdict2__merge_dict': {
+                'item2': 'test',
+                'list_item': ['test2']
+            }
+        })
+
+        self.assertEqual(results, [
+            {
+                'item1': 'test',
+                'item2': 'test',
+                'list_item': ['test1', 'test2']
+            }
+        ])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[{'initial_item': 'random value', 'list_item': ['test0']},
+                                                            'ignore', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
+                                                    {'item2': 'test', 'list_item': ['test2']}])
+    def test_merge_dict_with_initial_value(self, mock_set_options, mock_get_option, mock_template):
+        results = self.merge_vars_lookup.run(['__merge_dict'], {
+            'testdict1__merge_dict': {
+                'item1': 'test',
+                'list_item': ['test1']
+            },
+            'testdict2__merge_dict': {
+                'item2': 'test',
+                'list_item': ['test2']
+            }
+        })
+
+        self.assertEqual(results, [
+            {
+                'initial_item': 'random value',
+                'item1': 'test',
+                'item2': 'test',
+                'list_item': ['test0', 'test1', 'test2']
+            }
+        ])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'warn', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}])
+    @patch.object(Display, 'warning')
+    def test_merge_dict_non_unique_warning(self, mock_set_options, mock_get_option, mock_template, mock_display):
+        results = self.merge_vars_lookup.run(['__merge_non_unique'], {
+            'testdict1__merge_non_unique': {'item': 'value1'},
+            'testdict2__merge_non_unique': {'item': 'value2'}
+        })
+
+        self.assertTrue(mock_display.called)
+        self.assertEqual(results, [{'item': 'value2'}])
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'error', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[{'item': 'value1'}, {'item': 'value2'}])
+    def test_merge_dict_non_unique_error(self, mock_set_options, mock_get_option, mock_template):
+        with self.assertRaises(AnsibleError):
+            self.merge_vars_lookup.run(['__merge_non_unique'], {
+                'testdict1__merge_non_unique': {'item': 'value1'},
+                'testdict2__merge_non_unique': {'item': 'value2'}
+            })
+
+    @patch.object(AnsiblePlugin, 'set_options')
+    @patch.object(AnsiblePlugin, 'get_option', side_effect=[None, 'ignore', 'suffix'])
+    @patch.object(Templar, 'template', side_effect=[{'item1': 'test', 'list_item': ['test1']},
+                                                    ['item2', 'item3']])
+    def test_merge_list_and_dict(self, mock_set_options, mock_get_option, mock_template):
+        with self.assertRaises(AnsibleError):
+            self.merge_vars_lookup.run(['__merge_var'], {
+                'testlist__merge_var': {
+                    'item1': 'test',
+                    'list_item': ['test1']
+                },
+                'testdict__merge_var': ['item2', 'item3']
+            })