ModuleHelper variables management (#2162)

* added metadata for variables in module helper

* adjustments

* added separate support for tracking changes

* rewrote the diff code

* added integration test for module_helper

* using ansible.module_utils.common.dict_transformations.dict_merge

* improved dependency management

* restore ModuleHelper to base classes of CmdStateModuleHelper

* added assertions to ensure the failing module name appears in the error messages

* added test code for state-based modules

* fixed test name

* renamed class to VarMeta

* small fixes

* fixes from the PR

* fixed VarDict.__set_attr__

* added VarDict.__getitem__()

* added changelog fragment

* adjustments per PR

* ModuleHelper.output is now aware of conflicting variable names

* Update plugins/module_utils/module_helper.py

Co-authored-by: Felix Fontein <felix@fontein.de>

Co-authored-by: Felix Fontein <felix@fontein.de>

tests/integration/targets/module_helper/aliases

@@ -0,0 +1 @@
+shippable/posix/group4

tests/integration/targets/module_helper/library/mdepfail.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# (c) 2021, Alexei Znamensky <russoz@gmail.com>
+#
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+module: mdepfail
+author: "Alexei Znamensky (@russoz)"
+short_description: Simple module for testing
+description:
+  - Simple module test description.
+options:
+  a:
+    description: aaaa
+    type: int
+  b:
+    description: bbbb
+    type: str
+  c:
+    description: cccc
+    type: str
+'''
+
+EXAMPLES = ""
+
+RETURN = ""
+
+from ansible_collections.community.general.plugins.module_utils.module_helper import ModuleHelper
+from ansible.module_utils.basic import missing_required_lib
+
+with ModuleHelper.dependency("nopackagewiththisname", missing_required_lib("nopackagewiththisname")):
+    import nopackagewiththisname
+
+
+class MSimple(ModuleHelper):
+    output_params = ('a', 'b', 'c')
+    module = dict(
+        argument_spec=dict(
+            a=dict(type='int'),
+            b=dict(type='str'),
+            c=dict(type='str'),
+        ),
+    )
+
+    def __init_module__(self):
+        self.vars.set('value', None)
+        self.vars.set('abc', "abc", diff=True)
+
+    def __run__(self):
+        if (0 if self.vars.a is None else self.vars.a) >= 100:
+            raise Exception("a >= 100")
+        if self.vars.c == "abc change":
+            self.vars['abc'] = "changed abc"
+        if self.vars.get('a', 0) == 2:
+            self.vars['b'] = str(self.vars.b) * 2
+            self.vars['c'] = str(self.vars.c) * 2
+
+
+def main():
+    msimple = MSimple()
+    msimple.run()
+
+
+if __name__ == '__main__':
+    main()

tests/integration/targets/module_helper/library/msimple.py

@@ -0,0 +1,65 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# (c) 2021, Alexei Znamensky <russoz@gmail.com>
+#
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+module: msimple
+author: "Alexei Znamensky (@russoz)"
+short_description: Simple module for testing
+description:
+  - Simple module test description.
+options:
+  a:
+    description: aaaa
+    type: int
+  b:
+    description: bbbb
+    type: str
+  c:
+    description: cccc
+    type: str
+'''
+
+EXAMPLES = ""
+
+RETURN = ""
+
+from ansible_collections.community.general.plugins.module_utils.module_helper import ModuleHelper
+
+
+class MSimple(ModuleHelper):
+    output_params = ('a', 'b', 'c')
+    module = dict(
+        argument_spec=dict(
+            a=dict(type='int'),
+            b=dict(type='str'),
+            c=dict(type='str'),
+        ),
+    )
+
+    def __init_module__(self):
+        self.vars.set('value', None)
+        self.vars.set('abc', "abc", diff=True)
+
+    def __run__(self):
+        if (0 if self.vars.a is None else self.vars.a) >= 100:
+            raise Exception("a >= 100")
+        if self.vars.c == "abc change":
+            self.vars['abc'] = "changed abc"
+        if self.vars.get('a', 0) == 2:
+            self.vars['b'] = str(self.vars.b) * 2
+            self.vars['c'] = str(self.vars.c) * 2
+
+
+def main():
+    msimple = MSimple()
+    msimple.run()
+
+
+if __name__ == '__main__':
+    main()

tests/integration/targets/module_helper/library/mstate.py

@@ -0,0 +1,77 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# (c) 2021, Alexei Znamensky <russoz@gmail.com>
+#
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+module: mstate
+author: "Alexei Znamensky (@russoz)"
+short_description: State-based module for testing
+description:
+  - State-based module test description.
+options:
+  a:
+    description: aaaa
+    type: int
+    required: yes
+  b:
+    description: bbbb
+    type: str
+  c:
+    description: cccc
+    type: str
+  state:
+    description: test states
+    type: str
+    choices: [join, b_x_a, c_x_a, both_x_a]
+    default: join
+'''
+
+EXAMPLES = ""
+
+RETURN = ""
+
+from ansible_collections.community.general.plugins.module_utils.module_helper import StateModuleHelper
+
+
+class MState(StateModuleHelper):
+    output_params = ('a', 'b', 'c', 'state')
+    module = dict(
+        argument_spec=dict(
+            a=dict(type='int', required=True),
+            b=dict(type='str'),
+            c=dict(type='str'),
+            state=dict(type='str', choices=['join', 'b_x_a', 'c_x_a', 'both_x_a', 'nop'], default='join'),
+        ),
+    )
+
+    def __init_module__(self):
+        self.vars.set('result', "abc", diff=True)
+
+    def state_join(self):
+        self.vars['result'] = "".join([str(self.vars.a), str(self.vars.b), str(self.vars.c)])
+
+    def state_b_x_a(self):
+        self.vars['result'] = str(self.vars.b) * self.vars.a
+
+    def state_c_x_a(self):
+        self.vars['result'] = str(self.vars.c) * self.vars.a
+
+    def state_both_x_a(self):
+        self.vars['result'] = (str(self.vars.b) + str(self.vars.c)) * self.vars.a
+
+    def state_nop(self):
+        pass
+
+
+def main():
+    mstate = MState()
+    mstate.run()
+
+
+if __name__ == '__main__':
+    main()

tests/integration/targets/module_helper/tasks/main.yml

@@ -0,0 +1,3 @@
+- include_tasks: msimple.yml
+- include_tasks: mdepfail.yml
+- include_tasks: mstate.yml

tests/integration/targets/module_helper/tasks/mdepfail.yml

@@ -0,0 +1,14 @@
+- name: test failing dependency
+  mdepfail:
+    a: 123
+  ignore_errors: yes
+  register: result
+
+- name: assert failing dependency
+  assert:
+    that:
+      - result.failed is true
+      - '"Failed to import" in result.msg'
+      - '"nopackagewiththisname" in result.msg'
+      - '"ModuleNotFoundError:" in result.exception'
+      - '"nopackagewiththisname" in result.exception'

tests/integration/targets/module_helper/tasks/msimple.yml

@@ -0,0 +1,54 @@
+- name: test msimple 1
+  msimple:
+    a: 80
+  register: simple1
+
+- name: assert simple1
+  assert:
+    that:
+      - simple1.a == 80
+      - simple1.abc == "abc"
+      - simple1.changed is false
+      - simple1.value is none
+
+- name: test msimple 2
+  msimple:
+    a: 101
+  ignore_errors: yes
+  register: simple2
+
+- name: assert simple2
+  assert:
+    that:
+      - simple2.a == 101
+      - 'simple2.msg == "Module failed with exception: a >= 100"'
+      - simple2.abc == "abc"
+      - simple2.failed is true
+      - simple2.changed is false
+      - simple2.value is none
+
+- name: test msimple 3
+  msimple:
+    a: 2
+    b: potatoes
+  register: simple3
+
+- name: assert simple3
+  assert:
+    that:
+      - simple3.a == 2
+      - simple3.b == "potatoespotatoes"
+      - simple3.c == "NoneNone"
+      - simple3.changed is false
+
+- name: test msimple 4
+  msimple:
+    c: abc change
+  register: simple4
+
+- name: assert simple4
+  assert:
+    that:
+      - simple4.c == "abc change"
+      - simple4.abc == "changed abc"
+      - simple4.changed is true

tests/integration/targets/module_helper/tasks/mstate.yml

@@ -0,0 +1,79 @@
+- name: test mstate 1
+  mstate:
+    a: 80
+    b: banana
+    c: cashew
+    state: nop
+  register: state1
+
+- name: assert state1
+  assert:
+    that:
+      - state1.a == 80
+      - state1.b == "banana"
+      - state1.c == "cashew"
+      - state1.result == "abc"
+      - state1.changed is false
+
+- name: test mstate 2
+  mstate:
+    a: 80
+    b: banana
+    c: cashew
+  register: state2
+
+- name: assert state2
+  assert:
+    that:
+      - state2.a == 80
+      - state2.b == "banana"
+      - state2.c == "cashew"
+      - state2.result == "80bananacashew"
+      - state2.changed is true
+
+- name: test mstate 3
+  mstate:
+    a: 3
+    b: banana
+    state: b_x_a
+  register: state3
+
+- name: assert state3
+  assert:
+    that:
+      - state3.a == 3
+      - state3.b == "banana"
+      - state3.result == "bananabananabanana"
+      - state3.changed is true
+
+- name: test mstate 4
+  mstate:
+    a: 4
+    c: cashew
+    state: c_x_a
+  register: state4
+
+- name: assert state4
+  assert:
+    that:
+      - state4.a == 4
+      - state4.c == "cashew"
+      - state4.result == "cashewcashewcashewcashew"
+      - state4.changed is true
+
+- name: test mstate 5
+  mstate:
+    a: 5
+    b: foo
+    c: bar
+    state: both_x_a
+  register: state5
+
+- name: assert state5
+  assert:
+    that:
+      - state5.a == 5
+      - state5.b == "foo"
+      - state5.c == "bar"
+      - state5.result == "foobarfoobarfoobarfoobarfoobar"
+      - state5.changed is true

tests/unit/plugins/module_utils/test_module_helper.py

@@ -9,7 +9,7 @@ __metaclass__ = type
 import pytest
 
 from ansible_collections.community.general.plugins.module_utils.module_helper import (
-    ArgFormat, DependencyCtxMgr, ModuleHelper
+    ArgFormat, DependencyCtxMgr, ModuleHelper, VarMeta
 )
 
 
@@ -105,3 +105,58 @@ def test_dependency_ctxmgr():
     with ctx:
         import sys
     assert ctx.has_it
+
+
+def test_variable_meta():
+    meta = VarMeta()
+    assert meta.output is False
+    assert meta.diff is False
+    assert meta.value is None
+    meta.set_value("abc")
+    assert meta.initial_value == "abc"
+    assert meta.value == "abc"
+    assert meta.diff_result is None
+    meta.set_value("def")
+    assert meta.initial_value == "abc"
+    assert meta.value == "def"
+    assert meta.diff_result is None
+
+
+def test_variable_meta_diff():
+    meta = VarMeta(diff=True)
+    assert meta.output is False
+    assert meta.diff is True
+    assert meta.value is None
+    meta.set_value("abc")
+    assert meta.initial_value == "abc"
+    assert meta.value == "abc"
+    assert meta.diff_result is None
+    meta.set_value("def")
+    assert meta.initial_value == "abc"
+    assert meta.value == "def"
+    assert meta.diff_result == {"before": "abc", "after": "def"}
+    meta.set_value("ghi")
+    assert meta.initial_value == "abc"
+    assert meta.value == "ghi"
+    assert meta.diff_result == {"before": "abc", "after": "ghi"}
+
+
+def test_vardict():
+    vd = ModuleHelper.VarDict()
+    vd.set('a', 123)
+    assert vd['a'] == 123
+    assert vd.a == 123
+    assert 'a' in vd._meta
+    assert vd.meta('a').output is True
+    assert vd.meta('a').diff is False
+    assert vd.meta('a').change is False
+    vd['b'] = 456
+    vd.set_meta('a', diff=True, change=True)
+    vd.set_meta('b', diff=True, output=False)
+    vd['c'] = 789
+    vd['a'] = 'new_a'
+    vd['c'] = 'new_c'
+    assert vd.a == 'new_a'
+    assert vd.c == 'new_c'
+    assert vd.output() == {'a': 'new_a', 'c': 'new_c'}
+    assert vd.diff() == {'before': {'a': 123}, 'after': {'a': 'new_a'}}, "diff={0}".format(vd.diff())