pacemaker: add cluster maintenance mode checks (#10194)

* feat(maintenance): Add cluster maintenance mode checks for pacemaker

* bug(fix): Correct substring typo and unit test

This commit corrects a substring check for determining if the pacemaker
cluster is in maintenance mode. Additionally, unit test is corrected
with correct output from pacemaker when in maintenance mode.

* feat(maintenance): Add force argument for absent resources

This commit adds in --force argument for resources intended to be absent
within a cluster that is in maintenance mode. Without this argument, the
cluster will not attempt to remove the resource due to maintenance mode.
The resource will be declared as orphaned and exiting maintenance mode
will allow the cluster to remove the resource completely.

* refactor(review): Apply code review changes

This commit adds refactors to enhance code quality.

* doc(changelog): Add fragment for maintenance mode addition

tests/unit/plugins/modules/test_pacemaker_resource.yaml

@@ -30,6 +30,16 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 1
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
           err: ""
         - command: [/testbin/pcs, resource, create, virtual-ip, IPaddr2, "ip=[192.168.2.1]", --wait=300]
           environ: *env-def
@@ -60,6 +70,16 @@ test_cases:
           rc: 0
           out: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Started"
           err: ""
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 1
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
+          err: ""
         - command: [/testbin/pcs, resource, create, virtual-ip, IPaddr2, "ip=[192.168.2.1]", --wait=300]
           environ: *env-def
           rc: 1
@@ -70,6 +90,46 @@ test_cases:
           rc: 0
           out: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Started"
           err: ""
+  - id: test_present_minimal_input_resource_maintenance_mode
+    input:
+      state: present
+      name: virtual-ip
+      resource_type:
+        resource_name: IPaddr2
+      resource_option:
+        - "ip=[192.168.2.1]"
+    output:
+      changed: true
+      previous_value: null
+      value: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Stopped"
+    mocks:
+      run_command:
+        - command: [/testbin/pcs, resource, status, virtual-ip]
+          environ: *env-def
+          rc: 1
+          out: ""
+          err: ""
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 0
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
+            maintenance-mode=true
+          err: ""
+        - command: [/testbin/pcs, resource, create, virtual-ip, IPaddr2, "ip=[192.168.2.1]", --wait=300]
+          environ: *env-def
+          rc: 1
+          out: ""
+          err: "Error: resource 'virtual-ip' is not running on any node"
+        - command: [/testbin/pcs, resource, status, virtual-ip]
+          environ: *env-def
+          rc: 0
+          out: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Stopped"
+          err: ""
   - id: test_absent_minimal_input_resource_not_exist
     input:
       state: absent
@@ -84,6 +144,16 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 1
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
           err: ""
         - command: [/testbin/pcs, resource, remove, virtual-ip]
           environ: *env-def
@@ -94,7 +164,7 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
-          err: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
   - id: test_absent_minimal_input_resource_exists
     input:
       state: absent
@@ -110,6 +180,16 @@ test_cases:
           rc: 0
           out: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Started"
           err: ""
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 1
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
+          err: ""
         - command: [/testbin/pcs, resource, remove, virtual-ip]
           environ: *env-def
           rc: 0
@@ -119,7 +199,43 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
+  - id: test_absent_minimal_input_maintenance_mode
+    input:
+      state: absent
+      name: virtual-ip
+    output:
+      changed: true
+      previous_value: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Started"
+      value: null
+    mocks:
+      run_command:
+        - command: [/testbin/pcs, resource, status, virtual-ip]
+          environ: *env-def
+          rc: 0
+          out: "  * virtual-ip\t(ocf:heartbeat:IPAddr2):\t Started"
           err: ""
+        - command: [/testbin/pcs, property, config]
+          environ: *env-def
+          rc: 0
+          out: |
+            Cluster Properties: cib-bootstrap-options
+            cluster-infrastructure=corosync
+            cluster-name=hacluster
+            dc-version=2.1.9-1.fc41-7188dbf
+            have-watchdog=false
+            maintenance-mode=true
+          err: ""
+        - command: [/testbin/pcs, resource, remove, virtual-ip, --force]
+          environ: *env-def
+          rc: 0
+          out: ""
+          err: "Deleting Resource (and group) - virtual-ip"
+        - command: [/testbin/pcs, resource, status, virtual-ip]
+          environ: *env-def
+          rc: 1
+          out: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
   - id: test_enabled_minimal_input_resource_not_exists
     input:
       state: enabled
@@ -133,7 +249,7 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
-          err: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
         - command: [/testbin/pcs, resource, enable, virtual-ip]
           environ: *env-def
           rc: 1
@@ -177,7 +293,7 @@ test_cases:
           environ: *env-def
           rc: 1
           out: ""
-          err: ""
+          err: "Error: resource or tag id 'virtual-ip' not found"
         - command: [/testbin/pcs, resource, disable, virtual-ip]
           environ: *env-def
           rc: 1