pacemaker: add cluster maintenance mode checks (#10194)

* feat(maintenance): Add cluster maintenance mode checks for pacemaker

* bug(fix): Correct substring typo and unit test

This commit corrects a substring check for determining if the pacemaker
cluster is in maintenance mode. Additionally, unit test is corrected
with correct output from pacemaker when in maintenance mode.

* feat(maintenance): Add force argument for absent resources

This commit adds in --force argument for resources intended to be absent
within a cluster that is in maintenance mode. Without this argument, the
cluster will not attempt to remove the resource due to maintenance mode.
The resource will be declared as orphaned and exiting maintenance mode
will allow the cluster to remove the resource completely.

* refactor(review): Apply code review changes

This commit adds refactors to enhance code quality.

* doc(changelog): Add fragment for maintenance mode addition

changelogs/fragments/10194-add-pcs-resource-maintenace-mode.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - pacemaker_resource - add maintenance mode support for handling resource creation and deletion (https://github.com/ansible-collections/community.general/issues/10180, https://github.com/ansible-collections/community.general/pull/10194).

plugins/module_utils/pacemaker.py

@@ -37,10 +37,20 @@ def fmt_resource_argument(value):
     return ['--group' if value['argument_action'] == 'group' else value['argument_action']] + value['argument_option']
 
 
-def pacemaker_runner(module, cli_action, **kwargs):
+def get_pacemaker_maintenance_mode(runner):
+    with runner("config") as ctx:
+        rc, out, err = ctx.run()
+        maintenance_mode_output = list(filter(lambda string: "maintenance-mode=true" in string.lower(), out.splitlines()))
+        return bool(maintenance_mode_output)
+
+
+def pacemaker_runner(module, cli_action=None, **kwargs):
+    runner_command = ['pcs']
+    if cli_action:
+        runner_command.append(cli_action)
     runner = CmdRunner(
         module,
-        command=['pcs', cli_action],
+        command=runner_command,
         arg_formats=dict(
             state=cmd_runner_fmt.as_map(_state_map),
             name=cmd_runner_fmt.as_list(),
@@ -50,6 +60,8 @@ def pacemaker_runner(module, cli_action, **kwargs):
             resource_meta=cmd_runner_fmt.stack(cmd_runner_fmt.as_opt_val)("meta"),
             resource_argument=cmd_runner_fmt.as_func(fmt_resource_argument),
             wait=cmd_runner_fmt.as_opt_eq_val("--wait"),
+            config=cmd_runner_fmt.as_fixed("config"),
+            force=cmd_runner_fmt.as_bool("--force"),
         ),
         **kwargs
     )

plugins/modules/pacemaker_resource.py

@@ -135,7 +135,7 @@ cluster_resources:
 '''
 
 from ansible_collections.community.general.plugins.module_utils.module_helper import StateModuleHelper
-from ansible_collections.community.general.plugins.module_utils.pacemaker import pacemaker_runner
+from ansible_collections.community.general.plugins.module_utils.pacemaker import pacemaker_runner, get_pacemaker_maintenance_mode
 
 
 class PacemakerResource(StateModuleHelper):
@@ -168,6 +168,7 @@ class PacemakerResource(StateModuleHelper):
 
     def __init_module__(self):
         self.runner = pacemaker_runner(self.module, cli_action='resource')
+        self._maintenance_mode_runner = pacemaker_runner(self.module, cli_action='property')
         self.vars.set('previous_value', self._get())
         self.vars.set('value', self.vars.previous_value, change=True, diff=True)
 
@@ -184,8 +185,10 @@ class PacemakerResource(StateModuleHelper):
             return ctx.run(state='status')
 
     def state_absent(self):
-        with self.runner('state name', output_process=self._process_command_output(True, "does not exist"), check_mode_skip=True) as ctx:
-            ctx.run()
+        runner_args = ['state', 'name', 'force']
+        force = get_pacemaker_maintenance_mode(self._maintenance_mode_runner)
+        with self.runner(runner_args, output_process=self._process_command_output(True, "does not exist"), check_mode_skip=True) as ctx:
+            ctx.run(force=force)
             self.vars.set('value', self._get())
             self.vars.stdout = ctx.results_out
             self.vars.stderr = ctx.results_err
@@ -194,7 +197,7 @@ class PacemakerResource(StateModuleHelper):
     def state_present(self):
         with self.runner(
                 'state name resource_type resource_option resource_operation resource_meta resource_argument wait',
-                output_process=self._process_command_output(True, "already exists"),
+                output_process=self._process_command_output(not get_pacemaker_maintenance_mode(self._maintenance_mode_runner), "already exists"),
                 check_mode_skip=True) as ctx:
             ctx.run()
             self.vars.set('value', self._get())

tests/unit/plugins/modules/test_pacemaker_resource.yaml

@@ -30,6 +30,16 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 1
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
           err: ""
         - command: [/testbin/pcs, resource, create, virtual-ip, IPaddr2, "ip=[192.168.2.1]", --wait=300]
           environ: *env-def
@@ -60,6 +70,16 @@ test_cases:
           rc: 0
           out: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Started"
           err: ""
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 1
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
+          err: ""
         - command: [/testbin/pcs, resource, create, virtual-ip, IPaddr2, "ip=[192.168.2.1]", --wait=300]
           environ: *env-def
           rc: 1
@@ -70,6 +90,46 @@ test_cases:
           rc: 0
           out: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Started"
           err: ""
+  - id: test_present_minimal_input_resource_maintenance_mode
+    input:
+      state: present
+      name: virtual-ip
+      resource_type:
+        resource_name: IPaddr2
+      resource_option:
+        - "ip=[192.168.2.1]"
+    output:
+      changed: true
+      previous_value: null
+      value: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Stopped"
+    mocks:
+      run_command:
+        - command: [/testbin/pcs, resource, status, virtual-ip]
+          environ: *env-def
+          rc: 1
+          out: ""
+          err: ""
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 0
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
+            maintenance-mode=true
+          err: ""
+        - command: [/testbin/pcs, resource, create, virtual-ip, IPaddr2, "ip=[192.168.2.1]", --wait=300]
+          environ: *env-def
+          rc: 1
+          out: ""
+          err: "Error: resource 'virtual-ip' is not running on any node"
+        - command: [/testbin/pcs, resource, status, virtual-ip]
+          environ: *env-def
+          rc: 0
+          out: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Stopped"
+          err: ""
   - id: test_absent_minimal_input_resource_not_exist
     input:
       state: absent
@@ -84,6 +144,16 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 1
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
           err: ""
         - command: [/testbin/pcs, resource, remove, virtual-ip]
           environ: *env-def
@@ -94,7 +164,7 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
-          err: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
   - id: test_absent_minimal_input_resource_exists
     input:
       state: absent
@@ -110,6 +180,16 @@ test_cases:
           rc: 0
           out: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Started"
           err: ""
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 1
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
+          err: ""
         - command: [/testbin/pcs, resource, remove, virtual-ip]
           environ: *env-def
           rc: 0
@@ -119,7 +199,43 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
+  - id: test_absent_minimal_input_maintenance_mode
+    input:
+      state: absent
+      name: virtual-ip
+    output:
+      changed: true
+      previous_value: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Started"
+      value: null
+    mocks:
+      run_command:
+        - command: [/testbin/pcs, resource, status, virtual-ip]
+          environ: *env-def
+          rc: 0
+          out: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Started"
           err: ""
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 0
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
+            maintenance-mode=true
+          err: ""
+        - command: [/testbin/pcs, resource, remove, virtual-ip, --force]
+          environ: *env-def
+          rc: 0
+          out: ""
+          err: "Deleting Resource (and group) - virtual-ip"
+        - command: [/testbin/pcs, resource, status, virtual-ip]
+          environ: *env-def
+          rc: 1
+          out: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
   - id: test_enabled_minimal_input_resource_not_exists
     input:
       state: enabled
@@ -133,7 +249,7 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
-          err: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
         - command: [/testbin/pcs, resource, enable, virtual-ip]
           environ: *env-def
           rc: 1
@@ -177,7 +293,7 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
-          err: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
         - command: [/testbin/pcs, resource, disable, virtual-ip]
           environ: *env-def
           rc: 1