Adds aggregate functionality to the module (#51640)

lib/ansible/modules/network/f5/bigip_gtm_pool_member.py

@@ -48,6 +48,8 @@ options:
   pool:
     description:
       - Name of the GTM pool.
+      - For pools created on different partitions, you must specify partition of the pool in the full path format,
+        for example, C(/FooBar/pool_name).
     required: True
   partition:
     description:
@@ -77,6 +79,22 @@ options:
   description:
     description:
       - The description of the pool member.
+  aggregate:
+    description:
+      - List of GTM pool member definitions to be created, modified or removed.
+    aliases:
+      - members
+    version_added: 2.8
+  replace_all_with:
+    description:
+      - Remove members not defined in the C(aggregate) parameter.
+      - This operation is all or none, meaning that it will stop if there are some pool members
+        that cannot be removed.
+    default: no
+    type: bool
+    aliases:
+      - purge
+    version_added: 2.8
   limits:
     description:
       - Specifies resource thresholds or limit requirements at the pool member level.
@@ -157,6 +175,78 @@ EXAMPLES = r'''
       server: lb.mydomain.com
       user: admin
   delegate_to: localhost
+
+- name: Create a GTM pool member different partition
+  bigip_gtm_pool_member:
+    server_name: /Common/foo_name
+    virtual_server: GTMVSName
+    type: a
+    pool: /FooBar/foo-pool
+    partition: Common
+    provider:
+      password: secret
+      server: lb.mydomain.com
+      user: admin
+  delegate_to: localhost
+
+- name: Add GTM pool members aggregate
+  bigip_gtm_pool_member:
+    pool: pool1
+    type: a
+    aggregate:
+      - server_name: server1
+        virtual_server: vs1
+        partition: Common
+        port: 8080
+        description: web server1
+        member_order: 0
+      - server_name: server2
+        virtual_server: vs2
+        partition: Common
+        port: 8081
+        description: web server2
+        member_order: 1
+      - server_name: server3
+        virtual_server: vs3
+        partition: Common
+        port: 8082
+        description: web server3
+        member_order: 2
+    provider:
+      server: lb.mydomain.com
+      user: admin
+      password: secret
+  delegate_to: localhost
+
+- name: Add GTM pool members aggregate, remove non aggregates
+  bigip_gtm_pool_member:
+    pool: pool1
+    type: a
+    aggregate:
+      - server_name: server1
+        virtual_server: vs1
+        partition: Common
+        port: 8080
+        description: web server1
+        member_order: 0
+      - server_name: server2
+        virtual_server: vs2
+        partition: Common
+        port: 8081
+        description: web server2
+        member_order: 1
+      - server_name: server3
+        virtual_server: vs3
+        partition: Common
+        port: 8082
+        description: web server3
+        member_order: 2
+    replace_all_with: yes
+    provider:
+      server: lb.mydomain.com
+      user: admin
+      password: secret
+  delegate_to: localhost
 '''
 
 RETURN = r'''
@@ -220,10 +310,20 @@ description:
   returned: changed
   type: str
   sample: My description
+replace_all_with:
+  description: Purges all non-aggregate pool members from device
+  returned: changed
+  type: bool
+  sample: yes
 '''
 
+from copy import deepcopy
+
+from ansible.module_utils.urls import urlparse
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.basic import env_fallback
+from ansible.module_utils.six import iteritems
+from ansible.module_utils.network.common.utils import remove_default_spec
 
 try:
     from library.module_utils.compat.ipaddress import ip_address
@@ -238,6 +338,7 @@ try:
     from library.module_utils.network.f5.common import fail_json
     from library.module_utils.network.f5.common import flatten_boolean
     from library.module_utils.network.f5.icontrol import module_provisioned
+    from library.module_utils.network.f5.icontrol import TransactionContextManager
 except ImportError:
     from ansible.module_utils.compat.ipaddress import ip_address
     from ansible.module_utils.network.f5.bigip import F5RestClient
@@ -251,6 +352,7 @@ except ImportError:
     from ansible.module_utils.network.f5.common import fail_json
     from ansible.module_utils.network.f5.common import flatten_boolean
     from ansible.module_utils.network.f5.icontrol import module_provisioned
+    from ansible.module_utils.network.f5.icontrol import TransactionContextManager
 
 
 class Parameters(AnsibleF5Parameters):
@@ -316,6 +418,14 @@ class Parameters(AnsibleF5Parameters):
 
 
 class ApiParameters(Parameters):
+    def name(self):
+        # We need to do this because BIGIP allows / in names of GTM VS, allowing and users create such names incorrectly
+        # Despite the fact that GTM server and GTM Virtual Server cannot be created outside the Common partition
+        if self._values['subPath'] is None:
+            return self._values['name']
+        result = self._values['subPath'] + self._values['name']
+        return result
+
     @property
     def enabled(self):
         if 'enabled' in self._values:
@@ -490,9 +600,12 @@ class ModuleManager(object):
     def __init__(self, *args, **kwargs):
         self.module = kwargs.get('module', None)
         self.client = kwargs.get('client', None)
-        self.want = ModuleParameters(params=self.module.params)
+        self.want = None
-        self.have = ApiParameters()
+        self.have = None
-        self.changes = UsableChanges()
+        self.changes = None
+        self.replace_all_with = False
+        self.purge_links = list()
+        self.on_device = None
 
     def _set_changed_options(self):
         changed = {}
@@ -520,20 +633,91 @@ class ModuleManager(object):
             return True
         return False
 
-    def should_update(self):
-        result = self._update_changed_options()
-        if result:
-            return True
-        return False
-
     def exec_module(self):
         if not module_provisioned(self.client, 'gtm'):
             raise F5ModuleError(
                 "GTM must be provisioned to use this module."
             )
+        wants = None
+        if self.module.params['replace_all_with']:
+            self.replace_all_with = True
+
+        if self.module.params['aggregate']:
+            wants = self.merge_defaults_for_aggregate(self.module.params)
+
+        result = dict()
+        changed = False
+
+        if self.replace_all_with and self.purge_links:
+            self.purge()
+            changed = True
+
+        if self.module.params['aggregate']:
+            result['aggregate'] = list()
+            for want in wants:
+                output = self.execute(want)
+                if output['changed']:
+                    changed = output['changed']
+                result['aggregate'].append(output)
+        else:
+            output = self.execute(self.module.params)
+            if output['changed']:
+                changed = output['changed']
+            result.update(output)
+        if changed:
+            result['changed'] = True
+        return result
+
+    def merge_defaults_for_aggregate(self, params):
+        defaults = deepcopy(params)
+        aggregate = defaults.pop('aggregate')
+
+        for i, j in enumerate(aggregate):
+            for k, v in iteritems(defaults):
+                if k != 'replace_all_with':
+                    if j.get(k, None) is None and v is not None:
+                        aggregate[i][k] = v
+
+        if self.replace_all_with:
+            self.compare_aggregate_names(aggregate)
+
+        return aggregate
+
+    def _combine_names(self, item):
+        server_name = transform_name(item['partition'], item['server_name'])
+        virtual_server = transform_name(name=item['virtual_server'])
+        result = '{0}:{1}'.format(server_name, virtual_server)
+        return result
+
+    def _transform_api_names(self, item):
+        if 'subPath' in item and ['subPath'] is None:
+            return item['name']
+        result = transform_name(item['fullPath'])
+        return result
+
+    def compare_aggregate_names(self, items):
+        on_device = self._read_purge_collection()
+
+        if not on_device:
+            return False
+
+        aggregates = [self._combine_names(item) for item in items]
+        collection = [self._transform_api_names(item) for item in on_device]
+
+        diff = set(collection) - set(aggregates)
+
+        if diff:
+            to_purge = [item['selfLink'] for item in on_device if self._transform_api_names(item) in diff]
+            self.purge_links = to_purge
+
+    def execute(self, params=None):
+        self.want = ModuleParameters(params=params)
+        self.have = ApiParameters()
+        self.changes = UsableChanges()
+
         changed = False
         result = dict()
-        state = self.want.state
+        state = params['state']
 
         if state in ['present', 'enabled', 'disabled']:
             changed = self.present()
@@ -561,21 +745,15 @@ class ModuleManager(object):
         else:
             return self.create()
 
-    def exists(self):
+    def absent(self):
-        uri = "https://{0}:{1}/mgmt/tm/gtm/pool/{2}/{3}/members/{4}".format(
+        if self.exists():
-            self.client.provider['server'],
+            return self.remove()
-            self.client.provider['server_port'],
-            self.want.type,
-            transform_name(self.want.partition, self.want.pool),
-            transform_name(self.want.partition, self.want.name),
-        )
-        resp = self.client.api.get(uri)
-        try:
-            response = resp.json()
-        except ValueError:
-            return False
-        if resp.status == 404 or 'code' in response and response['code'] == 404:
         return False
+
+    def purge(self):
+        if self.module.check_mode:
+            return True
+        self.purge_from_device()
         return True
 
     def update(self):
@@ -587,6 +765,12 @@ class ModuleManager(object):
         self.update_on_device()
         return True
 
+    def should_update(self):
+        result = self._update_changed_options()
+        if result:
+            return True
+        return False
+
     def remove(self):
         if self.module.check_mode:
             return True
@@ -606,6 +790,78 @@ class ModuleManager(object):
         self.create_on_device()
         return True
 
+    def exists(self):
+        if not self.pool_exist():
+            raise F5ModuleError('The specified GTM pool does not exist')
+
+        uri = "https://{0}:{1}/mgmt/tm/gtm/pool/{2}/{3}/members/{4}".format(
+            self.client.provider['server'],
+            self.client.provider['server_port'],
+            self.want.type,
+            transform_name(name=fq_name(self.want.partition, self.want.pool)),
+            transform_name(self.want.partition, self.want.name),
+        )
+        resp = self.client.api.get(uri)
+        try:
+            response = resp.json()
+        except ValueError:
+            return False
+        if resp.status == 404 or 'code' in response and response['code'] == 404:
+            return False
+        return True
+
+    def pool_exist(self):
+        if self.replace_all_with:
+            type = self.module.params['type']
+            pool_name = transform_name(name=fq_name(self.module.params['partition'], self.module.params['pool']))
+        else:
+            pool_name = transform_name(name=fq_name(self.want.partition, self.want.pool))
+            type = self.want.type
+
+        uri = "https://{0}:{1}/mgmt/tm/gtm/pool/{2}/{3}".format(
+            self.client.provider['server'],
+            self.client.provider['server_port'],
+            type,
+            pool_name
+
+        )
+        resp = self.client.api.get(uri)
+        try:
+            response = resp.json()
+        except ValueError:
+            return False
+        if resp.status == 404 or 'code' in response and response['code'] == 404:
+            return False
+        return True
+
+    def _read_purge_collection(self):
+        type = self.module.params['type']
+        pool_name = transform_name(name=fq_name(self.module.params['partition'], self.module.params['pool']))
+
+        uri = "https://{0}:{1}/mgmt/tm/gtm/pool/{2}/{3}/members".format(
+            self.client.provider['server'],
+            self.client.provider['server_port'],
+            type,
+            pool_name
+        )
+
+        query = '?$select=name,selfLink,fullPath,subPath'
+        resp = self.client.api.get(uri + query)
+
+        try:
+            response = resp.json()
+        except ValueError as ex:
+            raise F5ModuleError(str(ex))
+
+        if 'code' in response and response['code'] == 400:
+            if 'message' in response:
+                raise F5ModuleError(response['message'])
+            else:
+                raise F5ModuleError(resp.content)
+        if 'items' in response:
+            return response['items']
+        return []
+
     def create_on_device(self):
         params = self.changes.api_params()
         params['name'] = self.want.name
@@ -614,7 +870,7 @@ class ModuleManager(object):
             self.client.provider['server'],
             self.client.provider['server_port'],
             self.want.type,
-            transform_name(self.want.partition, self.want.pool),
+            transform_name(name=fq_name(self.want.partition, self.want.pool)),
         )
         resp = self.client.api.post(uri, json=params)
         try:
@@ -635,7 +891,7 @@ class ModuleManager(object):
             self.client.provider['server'],
             self.client.provider['server_port'],
             self.want.type,
-            transform_name(self.want.partition, self.want.pool),
+            transform_name(name=fq_name(self.want.partition, self.want.pool)),
             transform_name(self.want.partition, self.want.name),
         )
         resp = self.client.api.patch(uri, json=params)
@@ -650,17 +906,12 @@ class ModuleManager(object):
             else:
                 raise F5ModuleError(resp.content)
 
-    def absent(self):
-        if self.exists():
-            return self.remove()
-        return False
-
     def remove_from_device(self):
         uri = "https://{0}:{1}/mgmt/tm/gtm/pool/{2}/{3}/members/{4}".format(
             self.client.provider['server'],
             self.client.provider['server_port'],
             self.want.type,
-            transform_name(self.want.partition, self.want.pool),
+            transform_name(name=fq_name(self.want.partition, self.want.pool)),
             transform_name(self.want.partition, self.want.name),
         )
         response = self.client.api.delete(uri)
@@ -673,7 +924,7 @@ class ModuleManager(object):
             self.client.provider['server'],
             self.client.provider['server_port'],
             self.want.type,
-            transform_name(self.want.partition, self.want.pool),
+            transform_name(name=fq_name(self.want.partition, self.want.pool)),
             transform_name(self.want.partition, self.want.name),
         )
         resp = self.client.api.get(uri)
@@ -689,6 +940,40 @@ class ModuleManager(object):
                 raise F5ModuleError(resp.content)
         return ApiParameters(params=response)
 
+    def _prepare_links(self, collection):
+        # this is to ensure no duplicates are in the provided collection
+        no_dupes = list(set(collection))
+        links = list()
+        purge_paths = [urlparse(link).path for link in no_dupes]
+
+        for path in purge_paths:
+            link = "https://{0}:{1}{2}".format(
+                self.client.provider['server'],
+                self.client.provider['server_port'],
+                path
+            )
+            links.append(link)
+        return links
+
+    def purge_from_device(self):
+        links = self._prepare_links(self.purge_links)
+
+        with TransactionContextManager(self.client) as transact:
+            for link in links:
+                resp = transact.api.delete(link)
+
+                try:
+                    response = resp.json()
+                except ValueError as ex:
+                    raise F5ModuleError(str(ex))
+
+                if 'code' in response and response['code'] == 400:
+                    if 'message' in response:
+                        raise F5ModuleError(response['message'])
+                    else:
+                        raise F5ModuleError(resp.content)
+        return True
+
 
 class ArgumentSpec(object):
     def __init__(self):
@@ -696,14 +981,9 @@ class ArgumentSpec(object):
         self.types = [
             'a', 'aaaa', 'cname', 'mx', 'naptr', 'srv'
         ]
-        argument_spec = dict(
+        element_spec = dict(
-            pool=dict(required=True),
+            server_name=dict(),
-            server_name=dict(required=True),
+            virtual_server=dict(),
-            virtual_server=dict(required=True),
-            type=dict(
-                choices=self.types,
-                required=True
-            ),
             member_order=dict(type='int'),
             monitor=dict(),
             ratio=dict(type='int'),
@@ -726,11 +1006,57 @@ class ArgumentSpec(object):
             partition=dict(
                 default='Common',
                 fallback=(env_fallback, ['F5_PARTITION'])
+            ),
+
         )
+
+        aggregate_spec = deepcopy(element_spec)
+
+        # remove default in aggregate spec, to handle common arguments
+        remove_default_spec(aggregate_spec)
+
+        self.argument_spec = dict(
+            aggregate=dict(
+                type='list',
+                elements='dict',
+                options=aggregate_spec,
+                aliases=['members'],
+                required_one_of=[
+                    ['server_name', 'virtual_server']
+                ],
+                required_together=[
+                    ['server_name', 'virtual_server']
+                ]
+
+            ),
+            pool=dict(required=True),
+            type=dict(
+                choices=self.types,
+                required=True
+            ),
+            replace_all_with=dict(
+                type='bool',
+                aliases=['purge'],
+                default='no'
+            ),
+            partition=dict(
+                default='Common',
+                fallback=(env_fallback, ['F5_PARTITION'])
             )
-        self.argument_spec = {}
+
+        )
+        self.argument_spec.update(element_spec)
         self.argument_spec.update(f5_argument_spec)
-        self.argument_spec.update(argument_spec)
+        self.required_together = [
+            ['server_name', 'virtual_server']
+        ]
+        self.mutually_exclusive = [
+            ['server_name', 'aggregate'],
+            ['virtual_server', 'aggregate']
+        ]
+        self.required_one_of = [
+            ['server_name', 'virtual_server', 'aggregate']
+        ]
 
 
 def main():
@@ -739,6 +1065,9 @@ def main():
     module = AnsibleModule(
         argument_spec=spec.argument_spec,
         supports_check_mode=spec.supports_check_mode,
+        mutually_exclusive=spec.mutually_exclusive,
+        required_one_of=spec.required_one_of,
+        required_together=spec.required_together,
     )
 
     client = F5RestClient(**module.params)

test/units/modules/network/f5/test_bigip_gtm_pool_member.py

@@ -151,14 +151,19 @@ class TestManager(unittest.TestCase):
             monitor='tcp',
             member_order=2,
             partition='Common',
+            provider=dict(
                 server='localhost',
                 password='password',
                 user='admin'
+            )
         ))
 
         module = AnsibleModule(
             argument_spec=self.spec.argument_spec,
-            supports_check_mode=self.spec.supports_check_mode
+            supports_check_mode=self.spec.supports_check_mode,
+            mutually_exclusive=self.spec.mutually_exclusive,
+            required_one_of=self.spec.required_one_of,
+            required_together=self.spec.required_together,
         )
 
         # Override methods in the specific type of manager
@@ -170,3 +175,45 @@ class TestManager(unittest.TestCase):
         results = mm.exec_module()
 
         assert results['changed'] is True
+
+    def test_create_aggregate_pool_members(self, *args):
+        set_module_args(dict(
+            pool='fake_pool',
+            type='a',
+            aggregate=[
+                dict(
+                    server_name='my-name1',
+                    virtual_server='some-vs2',
+                    state='present',
+                    partition='Common',
+
+                ),
+                dict(
+                    server_name='my-name1',
+                    virtual_server='some-vs1',
+                    state='present',
+                    partition='Common'
+                )
+            ],
+            provider=dict(
+                server='localhost',
+                password='password',
+                user='admin'
+            )
+        ))
+
+        module = AnsibleModule(
+            argument_spec=self.spec.argument_spec,
+            supports_check_mode=self.spec.supports_check_mode,
+            mutually_exclusive=self.spec.mutually_exclusive,
+            required_one_of=self.spec.required_one_of,
+            required_together=self.spec.required_together,
+        )
+
+        mm = ModuleManager(module=module)
+        mm.create_on_device = Mock(return_value=True)
+        mm.exists = Mock(return_value=False)
+
+        results = mm.exec_module()
+
+        assert results['changed'] is True