feat(lookup/merge_variables): Add all hosts mode to collect configuration across multiple hosts (#7999)

* Add Feature to collect variables accross different hosts

* fix merging lists

* adjust unit tests

* lint fixes

* adjusting integration tests

* remove white spaces

* Update plugins/lookup/merge_variables.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/lookup/merge_variables.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/lookup/merge_variables.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* apply suggested changes to correctly handling the initial_value parameter, incl. additional test

* whitespace

---------

Co-authored-by: Alexander Petrenz <alexander.petrenz@posteo.de>
Co-authored-by: Felix Fontein <felix@fontein.de>

tests/integration/targets/lookup_merge_variables/runme.sh

@@ -11,3 +11,6 @@ ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
 ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
 ANSIBLE_MERGE_VARIABLES_PATTERN_TYPE=suffix \
     ansible-playbook test_with_env.yml "$@"
+
+ANSIBLE_LOG_PATH=/tmp/ansible-test-merge-variables \
+    ansible-playbook -i test_inventory_all_hosts.yml test_all_hosts.yml "$@"

tests/integration/targets/lookup_merge_variables/test_all_hosts.yml

@@ -0,0 +1,64 @@
+---
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Test merge_variables lookup plugin (multiple hosts)
+  hosts: host0
+  gather_facts: false
+  tasks:
+    - name: Test merge dicts via all group
+      delegate_to: localhost
+      vars:
+        merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_ex', pattern_type='suffix', groups=['all']) }}"
+      block:
+        - name: Test merge dicts via all group - Print the merged dict
+          ansible.builtin.debug:
+            msg: "{{ merged_dict }}"
+
+        - name: Test merge dicts via all group - Validate that the dict is complete
+          ansible.builtin.assert:
+            that:
+              - "(merged_dict.keys() | list | length) == 4"
+              - "'item1' in merged_dict"
+              - "'item2' in merged_dict"
+              - "'item3' in merged_dict"
+              - "'list_item' in merged_dict"
+              - "merged_dict.list_item | length == 3"
+    - name: Test merge dicts via two of three groups
+      delegate_to: localhost
+      vars:
+        merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_in', pattern_type='suffix', groups=['dummy1', 'dummy2']) }}"
+      block:
+        - name: Test merge dicts via two of three groups - Print the merged dict
+          ansible.builtin.debug:
+            msg: "{{ merged_dict }}"
+
+        - name: Test merge dicts via two of three groups - Validate that the dict is complete
+          ansible.builtin.assert:
+            that:
+              - "(merged_dict.keys() | list | length) == 3"
+              - "'item1' in merged_dict"
+              - "'item2' in merged_dict"
+              - "'list_item' in merged_dict"
+              - "merged_dict.list_item | length == 2"
+    - name: Test merge dicts via two of three groups with inital value
+      delegate_to: localhost
+      vars:
+        initial_dict:
+          initial: initial_value
+        merged_dict: "{{ lookup('community.general.merge_variables', '__merge_dict_in', initial_value=initial_dict, pattern_type='suffix', groups=['dummy1', 'dummy2']) }}"
+      block:
+        - name: Test merge dicts via two of three groups with inital value - Print the merged dict
+          ansible.builtin.debug:
+            msg: "{{ merged_dict }}"
+
+        - name: Test merge dicts via two of three groups with inital value - Validate that the dict is complete
+          ansible.builtin.assert:
+            that:
+              - "(merged_dict.keys() | list | length) == 4"
+              - "'item1' in merged_dict"
+              - "'item2' in merged_dict"
+              - "'list_item' in merged_dict"
+              - "merged_dict.list_item | length == 2"

tests/integration/targets/lookup_merge_variables/test_inventory_all_hosts.yml

@@ -0,0 +1,52 @@
+---
+# Copyright (c) 2020, Thales Netherlands
+# Copyright (c) 2021, Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+all:
+  hosts:
+    host0:
+    host1:
+      testdict1__merge_dict_ex:
+        item1: value1
+        list_item:
+          - test1
+
+      testdict2__merge_dict_ex:
+        item2: value2
+        list_item:
+          - test2
+
+      testdict__merge_dict_in:
+        item1: value1
+        list_item:
+          - test1
+    host2:
+      testdict3__merge_dict_ex:
+        item3: value3
+        list_item:
+          - test3
+
+      testdict__merge_dict_in:
+        item2: value2
+        list_item:
+          - test2
+
+    host3:
+      testdict__merge_dict_in:
+        item3: value3
+        list_item:
+          - test3
+
+dummy1:
+  hosts:
+    host1:
+
+dummy2:
+  hosts:
+    host2:
+
+dummy3:
+  hosts:
+    host3: