Add latest updates from FTD Ansible downstream repository. (#53638)

* Add latest updates from FTD Ansible downstream repository.
 - add a better implementation of the upsert operation;
 - add API version lookup functionality;
 - add filter which remove duplicated references from the list of references;
 - fix minor bugs.

* fix issues outlined by ansibot

* fix argument name for _check_enum_method

lib/ansible/module_utils/network/ftd/common.py

@@ -15,11 +15,11 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
-
 import re
 
 from ansible.module_utils._text import to_text
 from ansible.module_utils.common.collections import is_string
+from ansible.module_utils.six import iteritems
 
 INVALID_IDENTIFIER_SYMBOLS = r'[^a-zA-Z0-9_]'
 
@@ -65,7 +65,7 @@ def construct_ansible_facts(response, params):
         response_body = response['items'] if 'items' in response else response
         if params.get('register_as'):
             facts[params['register_as']] = response_body
-        elif 'name' in response_body and 'type' in response_body:
+        elif response_body.get('name') and response_body.get('type'):
             object_name = re.sub(INVALID_IDENTIFIER_SYMBOLS, '_', response_body['name'].lower())
             fact_name = '%s_%s' % (response_body['type'], object_name)
             facts[fact_name] = response_body
@@ -181,13 +181,58 @@ def equal_objects(d1, d2):
     """
     Checks whether two objects are equal. Ignores special object properties (e.g. 'id', 'version') and
     properties with None and empty values. In case properties contains a reference to the other object,
-    only object identities (ids and types) are checked.
+    only object identities (ids and types) are checked. Also, if an array field contains multiple references
+    to the same object, duplicates are ignored when comparing objects.
 
     :type d1: dict
     :type d2: dict
     :return: True if passed objects and their properties are equal. Otherwise, returns False.
     """
-    d1 = dict((k, d1[k]) for k in d1.keys() if k not in NON_COMPARABLE_PROPERTIES and d1[k])
-    d2 = dict((k, d2[k]) for k in d2.keys() if k not in NON_COMPARABLE_PROPERTIES and d2[k])
 
+    def prepare_data_for_comparison(d):
+        d = dict((k, d[k]) for k in d.keys() if k not in NON_COMPARABLE_PROPERTIES and d[k])
+        d = delete_ref_duplicates(d)
+        return d
+
+    d1 = prepare_data_for_comparison(d1)
+    d2 = prepare_data_for_comparison(d2)
     return equal_dicts(d1, d2, compare_by_reference=False)
+
+
+def delete_ref_duplicates(d):
+    """
+    Removes reference duplicates from array fields: if an array contains multiple items and some of
+    them refer to the same object, only unique references are preserved (duplicates are removed).
+
+    :param d: dict with data
+    :type d: dict
+    :return: dict without reference duplicates
+    """
+
+    def delete_ref_duplicates_from_list(refs):
+        if all(type(i) == dict and is_object_ref(i) for i in refs):
+            unique_refs = set()
+            unique_list = list()
+            for i in refs:
+                key = (i['id'], i['type'])
+                if key not in unique_refs:
+                    unique_refs.add(key)
+                    unique_list.append(i)
+
+            return list(unique_list)
+
+        else:
+            return refs
+
+    if not d:
+        return d
+
+    modified_d = {}
+    for k, v in iteritems(d):
+        if type(v) == list:
+            modified_d[k] = delete_ref_duplicates_from_list(v)
+        elif type(v) == dict:
+            modified_d[k] = delete_ref_duplicates(v)
+        else:
+            modified_d[k] = v
+    return modified_d

lib/ansible/module_utils/network/ftd/configuration.py

@@ -31,9 +31,19 @@ INVALID_UUID_ERROR_MESSAGE = "Validation failed due to an invalid UUID"
 DUPLICATE_NAME_ERROR_MESSAGE = "Validation failed due to a duplicate name"
 
 MULTIPLE_DUPLICATES_FOUND_ERROR = (
-    "Cannot add a new object. An object(s) with the same attributes exists."
-    "Multiple objects returned according to filters being specified. "
-    "Please specify more specific filters which can find exact object that caused duplication error")
+    "Multiple objects matching specified filters are found. "
+    "Please, define filters more precisely to match one object exactly."
+)
+DUPLICATE_ERROR = (
+    "Cannot add a new object. "
+    "An object with the same name but different parameters already exists."
+)
+ADD_OPERATION_NOT_SUPPORTED_ERROR = (
+    "Cannot add a new object while executing an upsert request. "
+    "Creation of objects with this type is not supported."
+)
+
+PATH_PARAMS_FOR_DEFAULT_OBJ = {'objId': 'default'}
 
 PATH_PARAMS_FOR_DEFAULT_OBJ = {'objId': 'default'}
 
@@ -185,15 +195,10 @@ class OperationChecker(object):
         :return: True if all criteria required to provide requested called operation are satisfied, otherwise False
         :rtype: bool
         """
-        amount_operations_need_for_upsert_operation = 3
-        amount_supported_operations = 0
-        for operation_name, operation_spec in operations.items():
-            if cls.is_add_operation(operation_name, operation_spec) \
-                    or cls.is_edit_operation(operation_name, operation_spec) \
-                    or cls.is_get_list_operation(operation_name, operation_spec):
-                amount_supported_operations += 1
-
-        return amount_supported_operations == amount_operations_need_for_upsert_operation
+        has_edit_op = next((name for name, spec in iteritems(operations) if cls.is_edit_operation(name, spec)), None)
+        has_get_list_op = next((name for name, spec in iteritems(operations)
+                                if cls.is_get_list_operation(name, spec)), None)
+        return has_edit_op and has_get_list_op
 
 
 class BaseConfigurationResource(object):
@@ -264,8 +269,6 @@ class BaseConfigurationResource(object):
         return self._models_operations_specs_cache[model_name]
 
     def get_objects_by_filter(self, operation_name, params):
-        def transform_filters_to_query_param(filter_params):
-            return ';'.join(['%s:%s' % (key, val) for key, val in sorted(iteritems(filter_params))])
 
         def match_filters(filter_params, obj):
             for k, v in iteritems(filter_params):
@@ -275,14 +278,15 @@ class BaseConfigurationResource(object):
 
         dummy, query_params, path_params = _get_user_params(params)
         # copy required params to avoid mutation of passed `params` dict
-        get_list_params = {ParamName.QUERY_PARAMS: dict(query_params), ParamName.PATH_PARAMS: dict(path_params)}
+        url_params = {ParamName.QUERY_PARAMS: dict(query_params), ParamName.PATH_PARAMS: dict(path_params)}
 
         filters = params.get(ParamName.FILTERS) or {}
-        if filters:
-            get_list_params[ParamName.QUERY_PARAMS][QueryParams.FILTER] = transform_filters_to_query_param(filters)
+        if QueryParams.FILTER not in url_params[ParamName.QUERY_PARAMS] and 'name' in filters:
+            # most endpoints only support filtering by name, so remaining `filters` are applied on returned objects
+            url_params[ParamName.QUERY_PARAMS][QueryParams.FILTER] = 'name:%s' % filters['name']
 
         item_generator = iterate_over_pageable_resource(
-            partial(self.send_general_request, operation_name=operation_name), get_list_params
+            partial(self.send_general_request, operation_name=operation_name), url_params
         )
         return (i for i in item_generator if match_filters(filters, i))
 
@@ -294,45 +298,51 @@ class BaseConfigurationResource(object):
             return self.send_general_request(operation_name, params)
         except FtdServerError as e:
             if is_duplicate_name_error(e):
-                return self._check_if_the_same_object(operation_name, params, e)
+                return self._check_equality_with_existing_object(operation_name, params, e)
             else:
                 raise e
 
-    def _check_if_the_same_object(self, operation_name, params, e):
+    def _check_equality_with_existing_object(self, operation_name, params, e):
         """
-        Special check used in the scope of 'add_object' operation, which can be requested as a standalone operation or
-        in the scope of 'upsert_object' operation. This method executed in case 'add_object' failed and should try to
-        find the object that caused "object duplicate" error. In case single object found and it's equal to one we are
-        trying to create - the existing object will be returned (attempt to have kind of idempotency for add action).
-        In the case when we got more than one object returned as a result of the request to API - it will be hard to
-        find exact duplicate so the exception will be raised.
+        Looks for an existing object that caused "object duplicate" error and
+        checks whether it corresponds to the one specified in `params`.
+
+        In case a single object is found and it is equal to one we are trying
+        to create, the existing object is returned.
+
+        When the existing object is not equal to the object being created or
+        several objects are returned, an exception is raised.
         """
         model_name = self.get_operation_spec(operation_name)[OperationField.MODEL_NAME]
-        get_list_operation = self._find_get_list_operation(model_name)
-        if get_list_operation:
-            data = params[ParamName.DATA]
-            if not params.get(ParamName.FILTERS):
-                params[ParamName.FILTERS] = {'name': data['name']}
+        existing_obj = self._find_object_matching_params(model_name, params)
 
-            existing_obj = None
-            existing_objs = self.get_objects_by_filter(get_list_operation, params)
-
-            for i, obj in enumerate(existing_objs):
-                if i > 0:
-                    raise FtdConfigurationError(MULTIPLE_DUPLICATES_FOUND_ERROR)
-                existing_obj = obj
-
-            if existing_obj is not None:
-                if equal_objects(existing_obj, data):
-                    return existing_obj
-                else:
-                    raise FtdConfigurationError(
-                        'Cannot add new object. '
-                        'An object with the same name but different parameters already exists.',
-                        existing_obj)
+        if existing_obj is not None:
+            if equal_objects(existing_obj, params[ParamName.DATA]):
+                return existing_obj
+            else:
+                raise FtdConfigurationError(DUPLICATE_ERROR, existing_obj)
 
         raise e
 
+    def _find_object_matching_params(self, model_name, params):
+        get_list_operation = self._find_get_list_operation(model_name)
+        if not get_list_operation:
+            return None
+
+        data = params[ParamName.DATA]
+        if not params.get(ParamName.FILTERS):
+            params[ParamName.FILTERS] = {'name': data['name']}
+
+        obj = None
+        filtered_objs = self.get_objects_by_filter(get_list_operation, params)
+
+        for i, obj in enumerate(filtered_objs):
+            if i > 0:
+                raise FtdConfigurationError(MULTIPLE_DUPLICATES_FOUND_ERROR)
+            obj = obj
+
+        return obj
+
     def _find_get_list_operation(self, model_name):
         operations = self.get_operation_specs_by_model_name(model_name) or {}
         return next((
@@ -373,9 +383,12 @@ class BaseConfigurationResource(object):
         return self.send_general_request(operation_name, params)
 
     def send_general_request(self, operation_name, params):
+        def stop_if_check_mode():
+            if self._check_mode:
+                raise CheckModeException()
+
         self.validate_params(operation_name, params)
-        if self._check_mode:
-            raise CheckModeException()
+        stop_if_check_mode()
 
         data, query_params, path_params = _get_user_params(params)
         op_spec = self.get_operation_spec(operation_name)
@@ -418,28 +431,14 @@ class BaseConfigurationResource(object):
         if report:
             raise ValidationError(report)
 
-    def is_upsert_operation_supported(self, op_name):
-        """
-        Checks if all operations required for upsert object operation are defined in 'operations'.
-
-        :param op_name: upsert operation name
-        :type op_name: str
-        :return: True if all criteria required to provide requested called operation are satisfied, otherwise False
-        :rtype: bool
-        """
-        model_name = _extract_model_from_upsert_operation(op_name)
-        operations = self.get_operation_specs_by_model_name(model_name)
-        return self._operation_checker.is_upsert_operation_supported(operations)
-
     @staticmethod
     def _get_operation_name(checker, operations):
-        for operation_name, op_spec in operations.items():
-            if checker(operation_name, op_spec):
-                return operation_name
-        raise FtdConfigurationError("Operation is not supported")
+        return next((op_name for op_name, op_spec in iteritems(operations) if checker(op_name, op_spec)), None)
 
     def _add_upserted_object(self, model_operations, params):
         add_op_name = self._get_operation_name(self._operation_checker.is_add_operation, model_operations)
+        if not add_op_name:
+            raise FtdConfigurationError(ADD_OPERATION_NOT_SUPPORTED_ERROR)
         return self.add_object(add_op_name, params)
 
     def _edit_upserted_object(self, model_operations, existing_object, params):
@@ -453,9 +452,9 @@ class BaseConfigurationResource(object):
 
     def upsert_object(self, op_name, params):
         """
-        The wrapper on top of add object operation, get a list of objects and edit object operations that implement
-        upsert object operation. As a result, the object will be created if the object does not exist, if a single
-        object exists with requested 'params' this object will be updated otherwise, Exception will be raised.
+        Updates an object if it already exists, or tries to create a new one if there is no
+        such object. If multiple objects match filter criteria, or add operation is not supported,
+        the exception is raised.
 
         :param op_name: upsert operation name
         :type op_name: str
@@ -464,18 +463,26 @@ class BaseConfigurationResource(object):
         :return: upserted object representation
         :rtype: dict
         """
-        if not self.is_upsert_operation_supported(op_name):
-            raise FtdInvalidOperationNameError(op_name)
 
-        model_name = _extract_model_from_upsert_operation(op_name)
+        def extract_and_validate_model():
+            model = op_name[len(OperationNamePrefix.UPSERT):]
+            if not self._conn.get_model_spec(model):
+                raise FtdInvalidOperationNameError(op_name)
+            return model
+
+        model_name = extract_and_validate_model()
         model_operations = self.get_operation_specs_by_model_name(model_name)
 
-        try:
+        if not self._operation_checker.is_upsert_operation_supported(model_operations):
+            raise FtdInvalidOperationNameError(op_name)
+
+        existing_obj = self._find_object_matching_params(model_name, params)
+        if existing_obj:
+            equal_to_existing_obj = equal_objects(existing_obj, params[ParamName.DATA])
+            return existing_obj if equal_to_existing_obj \
+                else self._edit_upserted_object(model_operations, existing_obj, params)
+        else:
             return self._add_upserted_object(model_operations, params)
-        except FtdConfigurationError as e:
-            if e.obj:
-                return self._edit_upserted_object(model_operations, e.obj, params)
-            raise e
 
 
 def _set_default(params, field_name, value):
@@ -491,10 +498,6 @@ def is_put_request(operation_spec):
     return operation_spec[OperationField.METHOD] == HTTPMethod.PUT
 
 
-def _extract_model_from_upsert_operation(op_name):
-    return op_name[len(OperationNamePrefix.UPSERT):]
-
-
 def _get_user_params(params):
     return params.get(ParamName.DATA) or {}, params.get(ParamName.QUERY_PARAMS) or {}, params.get(
         ParamName.PATH_PARAMS) or {}
@@ -527,8 +530,8 @@ def iterate_over_pageable_resource(resource_func, params):
 
         raise FtdUnexpectedResponse(
             "Get List of Objects Response from the server contains more objects than requested. "
-            "There are {0} item(s) in the response while {1} was(ere) requested".format(items_in_response,
-                                                                                        items_expected)
+            "There are {0} item(s) in the response while {1} was(ere) requested".format(
+                items_in_response, items_expected)
         )
 
     while True:

lib/ansible/module_utils/network/ftd/fdm_swagger_client.py

@@ -31,6 +31,7 @@ class OperationField:
     MODEL_NAME = 'modelName'
     DESCRIPTION = 'description'
     RETURN_MULTIPLE_ITEMS = 'returnMultipleItems'
+    TAGS = "tags"
 
 
 class SpecProp:
@@ -105,14 +106,16 @@ class FdmSwaggerParser:
         This method simplifies a swagger format, resolves a model name for each operation, and adds documentation for
         each operation and model if it is provided.
 
-        :param spec: An API specification in the swagger format, see <https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md>
+        :param spec: An API specification in the swagger format, see
+            <https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md>
         :type spec: dict
         :param spec: A documentation map containing descriptions for models, operations and operation parameters.
         :type docs: dict
         :rtype: dict
         :return:
         Ex.
-            The models field contains model definition from swagger see <#https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md#definitions>
+            The models field contains model definition from swagger see
+            <#https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md#definitions>
             {
                 'models':{
                     'model_name':{...},
@@ -170,6 +173,10 @@ class FdmSwaggerParser:
             SpecProp.MODEL_OPERATIONS: self._get_model_operations(operations)
         }
 
+    @property
+    def base_path(self):
+        return self._base_path
+
     def _get_model_operations(self, operations):
         model_operations = {}
         for operations_name, params in iteritems(operations):
@@ -186,7 +193,8 @@ class FdmSwaggerParser:
                     OperationField.METHOD: method,
                     OperationField.URL: self._base_path + url,
                     OperationField.MODEL_NAME: self._get_model_name(method, params),
-                    OperationField.RETURN_MULTIPLE_ITEMS: self._return_multiple_items(params)
+                    OperationField.RETURN_MULTIPLE_ITEMS: self._return_multiple_items(params),
+                    OperationField.TAGS: params.get(OperationField.TAGS, [])
                 }
                 if OperationField.PARAMETERS in params:
                     operation[OperationField.PARAMETERS] = self._get_rest_params(params[OperationField.PARAMETERS])
@@ -205,8 +213,10 @@ class FdmSwaggerParser:
             operation[OperationField.DESCRIPTION] = operation_docs.get(PropName.DESCRIPTION, '')
 
             if OperationField.PARAMETERS in operation:
-                param_descriptions = dict((p[PropName.NAME], p[PropName.DESCRIPTION])
-                                          for p in operation_docs.get(OperationField.PARAMETERS, {}))
+                param_descriptions = dict((
+                    (p[PropName.NAME], p[PropName.DESCRIPTION])
+                    for p in operation_docs.get(OperationField.PARAMETERS, {})
+                ))
 
                 for param_name, params_spec in operation[OperationField.PARAMETERS][OperationParams.PATH].items():
                     params_spec[OperationField.DESCRIPTION] = param_descriptions.get(param_name, '')
@@ -493,7 +503,7 @@ class FdmSwaggerValidator:
             if prop_name in params:
                 expected_type = prop[PropName.TYPE]
                 value = params[prop_name]
-                if prop_name in params and not self._is_correct_simple_types(expected_type, value):
+                if prop_name in params and not self._is_correct_simple_types(expected_type, value, allow_null=False):
                     self._add_invalid_type_report(status, '', prop_name, expected_type, value)
 
     def _validate_object(self, status, model, data, path):
@@ -505,9 +515,9 @@ class FdmSwaggerValidator:
     def _is_enum(self, model):
         return self._is_string_type(model) and PropName.ENUM in model
 
-    def _check_enum(self, status, model, value, path):
-        if value not in model[PropName.ENUM]:
-            self._add_invalid_type_report(status, path, '', PropName.ENUM, value)
+    def _check_enum(self, status, model, data, path):
+        if data is not None and data not in model[PropName.ENUM]:
+            self._add_invalid_type_report(status, path, '', PropName.ENUM, data)
 
     def _add_invalid_type_report(self, status, path, prop_name, expected_type, actually_value):
         status[PropName.INVALID_TYPE].append({
@@ -517,6 +527,9 @@ class FdmSwaggerValidator:
         })
 
     def _check_object(self, status, model, data, path):
+        if data is None:
+            return
+
         if not isinstance(data, dict):
             self._add_invalid_type_report(status, path, '', PropType.OBJECT, data)
             return None
@@ -550,12 +563,14 @@ class FdmSwaggerValidator:
 
     def _check_required_fields(self, status, required_fields, data, path):
         missed_required_fields = [self._create_path_to_field(path, field) for field in
-                                  required_fields if field not in data.keys()]
+                                  required_fields if field not in data.keys() or data[field] is None]
         if len(missed_required_fields) > 0:
             status[PropName.REQUIRED] += missed_required_fields
 
     def _check_array(self, status, model, data, path):
-        if not isinstance(data, list):
+        if data is None:
+            return
+        elif not isinstance(data, list):
             self._add_invalid_type_report(status, path, '', PropType.ARRAY, data)
         else:
             item_model = model[PropName.ITEMS]
@@ -564,7 +579,7 @@ class FdmSwaggerValidator:
                                   '')
 
     @staticmethod
-    def _is_correct_simple_types(expected_type, value):
+    def _is_correct_simple_types(expected_type, value, allow_null=True):
         def is_numeric_string(s):
             try:
                 float(s)
@@ -572,7 +587,9 @@ class FdmSwaggerValidator:
             except ValueError:
                 return False
 
-        if expected_type == PropType.STRING:
+        if value is None and allow_null:
+            return True
+        elif expected_type == PropType.STRING:
             return isinstance(value, string_types)
         elif expected_type == PropType.BOOLEAN:
             return isinstance(value, bool)