FTD modules: upsert functionality and bug fixes (#47747)

* FTD modules: bug fixes and upsert functionality

* Fix sanity checks

* Fix unit tests for Python 2.6

* Log status code for login/logout

* Use string formatting in logging

lib/ansible/module_utils/network/ftd/common.py

@@ -18,6 +18,9 @@
 
 import re
 
+from ansible.module_utils._text import to_text
+from ansible.module_utils.common.collections import is_string
+
 INVALID_IDENTIFIER_SYMBOLS = r'[^a-zA-Z0-9_]'
 
 IDENTITY_PROPERTIES = ['id', 'version', 'ruleId']
@@ -38,7 +41,10 @@ class ResponseParams:
 
 
 class FtdConfigurationError(Exception):
-    pass
+    def __init__(self, msg, obj=None):
+        super(FtdConfigurationError, self).__init__(msg)
+        self.msg = msg
+        self.obj = obj
 
 
 class FtdServerError(Exception):
@@ -48,6 +54,11 @@ class FtdServerError(Exception):
         self.code = code
 
 
+class FtdUnexpectedResponse(Exception):
+    """The exception to be raised in case of unexpected responses from 3d parties."""
+    pass
+
+
 def construct_ansible_facts(response, params):
     facts = dict()
     if response:
@@ -149,6 +160,11 @@ def equal_values(v1, v2):
     :return: True if types and content of passed values are equal. Otherwise, returns False.
     :rtype: bool
     """
+
+    # string-like values might have same text but different types, so checking them separately
+    if is_string(v1) and is_string(v2):
+        return to_text(v1) == to_text(v2)
+
     if type(v1) != type(v2):
         return False
     value_type = type(v1)

lib/ansible/module_utils/network/ftd/configuration.py

@@ -15,11 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
-
+import copy
 from functools import partial
 
-from ansible.module_utils.network.ftd.common import HTTPMethod, equal_objects, copy_identity_properties, \
-    FtdConfigurationError, FtdServerError, ResponseParams
+from ansible.module_utils.network.ftd.common import HTTPMethod, equal_objects, FtdConfigurationError, \
+    FtdServerError, ResponseParams, copy_identity_properties, FtdUnexpectedResponse
+from ansible.module_utils.network.ftd.fdm_swagger_client import OperationField, ValidationError
+from ansible.module_utils.six import iteritems
 
 DEFAULT_PAGE_SIZE = 10
 DEFAULT_OFFSET = 0
@@ -28,86 +30,358 @@ UNPROCESSABLE_ENTITY_STATUS = 422
 INVALID_UUID_ERROR_MESSAGE = "Validation failed due to an invalid UUID"
 DUPLICATE_NAME_ERROR_MESSAGE = "Validation failed due to a duplicate name"
 
+MULTIPLE_DUPLICATES_FOUND_ERROR = (
+    "Cannot add a new object. An object(s) with the same attributes exists."
+    "Multiple objects returned according to filters being specified. "
+    "Please specify more specific filters which can find exact object that caused duplication error")
+
+
+class OperationNamePrefix:
+    ADD = 'add'
+    EDIT = 'edit'
+    GET = 'get'
+    DELETE = 'delete'
+    UPSERT = 'upsert'
+
+
+class QueryParams:
+    FILTER = 'filter'
+
+
+class ParamName:
+    QUERY_PARAMS = 'query_params'
+    PATH_PARAMS = 'path_params'
+    DATA = 'data'
+    FILTERS = 'filters'
+
+
+class CheckModeException(Exception):
+    pass
+
+
+class FtdInvalidOperationNameError(Exception):
+    def __init__(self, operation_name):
+        super(FtdInvalidOperationNameError, self).__init__(operation_name)
+        self.operation_name = operation_name
+
+
+class OperationChecker(object):
+
+    @classmethod
+    def is_add_operation(cls, operation_name, operation_spec):
+        """
+        Check if operation defined with 'operation_name' is add object operation according to 'operation_spec'.
+
+        :param operation_name: name of the operation being called by the user
+        :type operation_name: str
+        :param operation_spec: specification of the operation being called by the user
+        :type operation_spec: dict
+        :return: True if the called operation is add object operation, otherwise False
+        :rtype: bool
+        """
+        # Some endpoints have non-CRUD operations, so checking operation name is required in addition to the HTTP method
+        return operation_name.startswith(OperationNamePrefix.ADD) and is_post_request(operation_spec)
+
+    @classmethod
+    def is_edit_operation(cls, operation_name, operation_spec):
+        """
+        Check if operation defined with 'operation_name' is edit object operation according to 'operation_spec'.
+
+        :param operation_name: name of the operation being called by the user
+        :type operation_name: str
+        :param operation_spec: specification of the operation being called by the user
+        :type operation_spec: dict
+        :return: True if the called operation is edit object operation, otherwise False
+        :rtype: bool
+        """
+        # Some endpoints have non-CRUD operations, so checking operation name is required in addition to the HTTP method
+        return operation_name.startswith(OperationNamePrefix.EDIT) and is_put_request(operation_spec)
+
+    @classmethod
+    def is_delete_operation(cls, operation_name, operation_spec):
+        """
+        Check if operation defined with 'operation_name' is delete object operation according to 'operation_spec'.
+
+        :param operation_name: name of the operation being called by the user
+        :type operation_name: str
+        :param operation_spec: specification of the operation being called by the user
+        :type operation_spec: dict
+        :return: True if the called operation is delete object operation, otherwise False
+        :rtype: bool
+        """
+        # Some endpoints have non-CRUD operations, so checking operation name is required in addition to the HTTP method
+        return operation_name.startswith(OperationNamePrefix.DELETE) \
+            and operation_spec[OperationField.METHOD] == HTTPMethod.DELETE
+
+    @classmethod
+    def is_get_list_operation(cls, operation_name, operation_spec):
+        """
+        Check if operation defined with 'operation_name' is get list of objects operation according to 'operation_spec'.
+
+        :param operation_name: name of the operation being called by the user
+        :type operation_name: str
+        :param operation_spec: specification of the operation being called by the user
+        :type operation_spec: dict
+        :return: True if the called operation is get a list of objects operation, otherwise False
+        :rtype: bool
+        """
+        return operation_spec[OperationField.METHOD] == HTTPMethod.GET \
+            and operation_spec[OperationField.RETURN_MULTIPLE_ITEMS]
+
+    @classmethod
+    def is_get_operation(cls, operation_name, operation_spec):
+        """
+        Check if operation defined with 'operation_name' is get objects operation according to 'operation_spec'.
+
+        :param operation_name: name of the operation being called by the user
+        :type operation_name: str
+        :param operation_spec: specification of the operation being called by the user
+        :type operation_spec: dict
+        :return: True if the called operation is get object operation, otherwise False
+        :rtype: bool
+        """
+        return operation_spec[OperationField.METHOD] == HTTPMethod.GET \
+            and not operation_spec[OperationField.RETURN_MULTIPLE_ITEMS]
+
+    @classmethod
+    def is_upsert_operation(cls, operation_name):
+        """
+        Check if operation defined with 'operation_name' is upsert objects operation according to 'operation_name'.
+
+        :param operation_name: name of the operation being called by the user
+        :type operation_name: str
+        :return: True if the called operation is upsert object operation, otherwise False
+        :rtype: bool
+        """
+        return operation_name.startswith(OperationNamePrefix.UPSERT)
+
+    @classmethod
+    def is_find_by_filter_operation(cls, operation_name, params, operation_spec):
+        """
+        Checks whether the called operation is 'find by filter'. This operation fetches all objects and finds
+        the matching ones by the given filter. As filtering is done on the client side, this operation should be used
+        only when selected filters are not implemented on the server side.
+
+        :param operation_name: name of the operation being called by the user
+        :type operation_name: str
+        :param operation_spec: specification of the operation being called by the user
+        :type operation_spec: dict
+        :param params: params - params should contain 'filters'
+        :return: True if the called operation is find by filter, otherwise False
+        :rtype: bool
+        """
+        is_get_list = cls.is_get_list_operation(operation_name, operation_spec)
+        return is_get_list and ParamName.FILTERS in params and params[ParamName.FILTERS]
+
+    @classmethod
+    def is_upsert_operation_supported(cls, operations):
+        """
+        Checks if all operations required for upsert object operation are defined in 'operations'.
+
+        :param operations: specification of the operations supported by model
+        :type operations: dict
+        :return: True if all criteria required to provide requested called operation are satisfied, otherwise False
+        :rtype: bool
+        """
+        amount_operations_need_for_upsert_operation = 3
+        amount_supported_operations = 0
+        for operation_name, operation_spec in operations.items():
+            if cls.is_add_operation(operation_name, operation_spec) \
+                    or cls.is_edit_operation(operation_name, operation_spec) \
+                    or cls.is_get_list_operation(operation_name, operation_spec):
+                amount_supported_operations += 1
+
+        return amount_supported_operations == amount_operations_need_for_upsert_operation
+
 
 class BaseConfigurationResource(object):
-    def __init__(self, conn):
+
+    def __init__(self, conn, check_mode=False):
         self._conn = conn
         self.config_changed = False
+        self._operation_spec_cache = {}
+        self._models_operations_specs_cache = {}
+        self._check_mode = check_mode
+        self._operation_checker = OperationChecker
 
-    def get_object_by_name(self, url_path, name, path_params=None):
-        item_generator = iterate_over_pageable_resource(
-            partial(self.send_request, url_path=url_path, http_method=HTTPMethod.GET, path_params=path_params),
-            {'filter': 'name:%s' % name}
-        )
-        # not all endpoints support filtering so checking name explicitly
-        return next((item for item in item_generator if item['name'] == name), None)
+    def execute_operation(self, op_name, params):
+        """
+        Allow user request execution of simple operations(natively supported by API provider) as well as complex
+        operations(operations that are implemented as a set of simple operations).
 
-    def get_objects_by_filter(self, url_path, filters, path_params=None, query_params=None):
-        def match_filters(obj):
-            for k, v in filters.items():
+        :param op_name: name of the operation being called by the user
+        :type op_name: str
+        :param params: definition of the params that operation should be executed with
+        :type params: dict
+        :return: Result of the operation being executed
+        :rtype: dict
+        """
+        if self._operation_checker.is_upsert_operation(op_name):
+            return self.upsert_object(op_name, params)
+        else:
+            return self.crud_operation(op_name, params)
+
+    def crud_operation(self, op_name, params):
+        """
+        Allow user request execution of simple operations(natively supported by API provider) only.
+
+        :param op_name: name of the operation being called by the user
+        :type op_name: str
+        :param params: definition of the params that operation should be executed with
+        :type params: dict
+        :return: Result of the operation being executed
+        :rtype: dict
+        """
+        op_spec = self.get_operation_spec(op_name)
+        if op_spec is None:
+            raise FtdInvalidOperationNameError(op_name)
+
+        if self._operation_checker.is_add_operation(op_name, op_spec):
+            resp = self.add_object(op_name, params)
+        elif self._operation_checker.is_edit_operation(op_name, op_spec):
+            resp = self.edit_object(op_name, params)
+        elif self._operation_checker.is_delete_operation(op_name, op_spec):
+            resp = self.delete_object(op_name, params)
+        elif self._operation_checker.is_find_by_filter_operation(op_name, params, op_spec):
+            resp = list(self.get_objects_by_filter(op_name, params))
+        else:
+            resp = self.send_general_request(op_name, params)
+        return resp
+
+    def get_operation_spec(self, operation_name):
+        if operation_name not in self._operation_spec_cache:
+            self._operation_spec_cache[operation_name] = self._conn.get_operation_spec(operation_name)
+        return self._operation_spec_cache[operation_name]
+
+    def get_operation_specs_by_model_name(self, model_name):
+        if model_name not in self._models_operations_specs_cache:
+            model_op_specs = self._conn.get_operation_specs_by_model_name(model_name)
+            self._models_operations_specs_cache[model_name] = model_op_specs
+            for op_name, op_spec in iteritems(model_op_specs):
+                self._operation_spec_cache.setdefault(op_name, op_spec)
+        return self._models_operations_specs_cache[model_name]
+
+    def get_objects_by_filter(self, operation_name, params):
+        def transform_filters_to_query_param(filter_params):
+            return ';'.join(['%s:%s' % (key, val) for key, val in sorted(iteritems(filter_params))])
+
+        def match_filters(filter_params, obj):
+            for k, v in iteritems(filter_params):
                 if k not in obj or obj[k] != v:
                     return False
             return True
 
-        item_generator = iterate_over_pageable_resource(
-            partial(self.send_request, url_path=url_path, http_method=HTTPMethod.GET, path_params=path_params),
-            query_params
-        )
-        return [i for i in item_generator if match_filters(i)]
+        dummy, query_params, path_params = _get_user_params(params)
+        # copy required params to avoid mutation of passed `params` dict
+        get_list_params = {ParamName.QUERY_PARAMS: dict(query_params), ParamName.PATH_PARAMS: dict(path_params)}
 
-    def add_object(self, url_path, body_params, path_params=None, query_params=None, update_if_exists=False):
+        filters = params.get(ParamName.FILTERS) or {}
+        if filters:
+            get_list_params[ParamName.QUERY_PARAMS][QueryParams.FILTER] = transform_filters_to_query_param(filters)
+
+        item_generator = iterate_over_pageable_resource(
+            partial(self.send_general_request, operation_name=operation_name), get_list_params
+        )
+        return (i for i in item_generator if match_filters(filters, i))
+
+    def add_object(self, operation_name, params):
         def is_duplicate_name_error(err):
             return err.code == UNPROCESSABLE_ENTITY_STATUS and DUPLICATE_NAME_ERROR_MESSAGE in str(err)
 
-        def update_existing_object(obj):
-            new_path_params = {} if path_params is None else path_params
-            new_path_params['objId'] = obj['id']
-            return self.send_request(url_path=url_path + '/{objId}',
-                                     http_method=HTTPMethod.PUT,
-                                     body_params=copy_identity_properties(obj, body_params),
-                                     path_params=new_path_params,
-                                     query_params=query_params)
-
         try:
-            return self.send_request(url_path=url_path, http_method=HTTPMethod.POST, body_params=body_params,
-                                     path_params=path_params, query_params=query_params)
+            return self.send_general_request(operation_name, params)
         except FtdServerError as e:
             if is_duplicate_name_error(e):
-                existing_obj = self.get_object_by_name(url_path, body_params['name'], path_params)
-                if equal_objects(existing_obj, body_params):
-                    return existing_obj
-                elif update_if_exists:
-                    return update_existing_object(existing_obj)
-                else:
-                    raise FtdConfigurationError(
-                        'Cannot add new object. An object with the same name but different parameters already exists.')
+                return self._check_if_the_same_object(operation_name, params, e)
             else:
                 raise e
 
-    def delete_object(self, url_path, path_params):
+    def _check_if_the_same_object(self, operation_name, params, e):
+        """
+        Special check used in the scope of 'add_object' operation, which can be requested as a standalone operation or
+        in the scope of 'upsert_object' operation. This method executed in case 'add_object' failed and should try to
+        find the object that caused "object duplicate" error. In case single object found and it's equal to one we are
+        trying to create - the existing object will be returned (attempt to have kind of idempotency for add action).
+        In the case when we got more than one object returned as a result of the request to API - it will be hard to
+        find exact duplicate so the exception will be raised.
+        """
+        model_name = self.get_operation_spec(operation_name)[OperationField.MODEL_NAME]
+        get_list_operation = self._find_get_list_operation(model_name)
+        if get_list_operation:
+            data = params[ParamName.DATA]
+            if not params.get(ParamName.FILTERS):
+                params[ParamName.FILTERS] = {'name': data['name']}
+
+            existing_obj = None
+            existing_objs = self.get_objects_by_filter(get_list_operation, params)
+
+            for i, obj in enumerate(existing_objs):
+                if i > 0:
+                    raise FtdConfigurationError(MULTIPLE_DUPLICATES_FOUND_ERROR)
+                existing_obj = obj
+
+            if existing_obj is not None:
+                if equal_objects(existing_obj, data):
+                    return existing_obj
+                else:
+                    raise FtdConfigurationError(
+                        'Cannot add new object. '
+                        'An object with the same name but different parameters already exists.',
+                        existing_obj)
+
+        raise e
+
+    def _find_get_list_operation(self, model_name):
+        operations = self.get_operation_specs_by_model_name(model_name) or {}
+        return next((
+            op for op, op_spec in operations.items()
+            if self._operation_checker.is_get_list_operation(op, op_spec)), None)
+
+    def _find_get_operation(self, model_name):
+        operations = self.get_operation_specs_by_model_name(model_name) or {}
+        return next((
+            op for op, op_spec in operations.items()
+            if self._operation_checker.is_get_operation(op, op_spec)), None)
+
+    def delete_object(self, operation_name, params):
         def is_invalid_uuid_error(err):
             return err.code == UNPROCESSABLE_ENTITY_STATUS and INVALID_UUID_ERROR_MESSAGE in str(err)
 
         try:
-            return self.send_request(url_path=url_path, http_method=HTTPMethod.DELETE, path_params=path_params)
+            return self.send_general_request(operation_name, params)
         except FtdServerError as e:
             if is_invalid_uuid_error(e):
                 return {'status': 'Referenced object does not exist'}
             else:
                 raise e
 
-    def edit_object(self, url_path, body_params, path_params=None, query_params=None):
-        existing_object = self.send_request(url_path=url_path, http_method=HTTPMethod.GET, path_params=path_params)
+    def edit_object(self, operation_name, params):
+        data, dummy, path_params = _get_user_params(params)
 
-        if not existing_object:
-            raise FtdConfigurationError('Referenced object does not exist')
-        elif equal_objects(existing_object, body_params):
-            return existing_object
-        else:
-            return self.send_request(url_path=url_path, http_method=HTTPMethod.PUT, body_params=body_params,
-                                     path_params=path_params, query_params=query_params)
+        model_name = self.get_operation_spec(operation_name)[OperationField.MODEL_NAME]
+        get_operation = self._find_get_operation(model_name)
 
-    def send_request(self, url_path, http_method, body_params=None, path_params=None, query_params=None):
+        if get_operation:
+            existing_object = self.send_general_request(get_operation, {ParamName.PATH_PARAMS: path_params})
+            if not existing_object:
+                raise FtdConfigurationError('Referenced object does not exist')
+            elif equal_objects(existing_object, data):
+                return existing_object
+
+        return self.send_general_request(operation_name, params)
+
+    def send_general_request(self, operation_name, params):
+        self.validate_params(operation_name, params)
+        if self._check_mode:
+            raise CheckModeException()
+
+        data, query_params, path_params = _get_user_params(params)
+        op_spec = self.get_operation_spec(operation_name)
+        url, method = op_spec[OperationField.URL], op_spec[OperationField.METHOD]
+
+        return self._send_request(url, method, data, path_params, query_params)
+
+    def _send_request(self, url_path, http_method, body_params=None, path_params=None, query_params=None):
         def raise_for_failure(resp):
             if not resp[ResponseParams.SUCCESS]:
                 raise FtdServerError(resp[ResponseParams.RESPONSE], resp[ResponseParams.STATUS_CODE])
@@ -119,28 +393,152 @@ class BaseConfigurationResource(object):
             self.config_changed = True
         return response[ResponseParams.RESPONSE]
 
+    def validate_params(self, operation_name, params):
+        report = {}
+        op_spec = self.get_operation_spec(operation_name)
+        data, query_params, path_params = _get_user_params(params)
 
-def iterate_over_pageable_resource(resource_func, query_params=None):
+        def validate(validation_method, field_name, user_params):
+            key = 'Invalid %s provided' % field_name
+            try:
+                is_valid, validation_report = validation_method(operation_name, user_params)
+                if not is_valid:
+                    report[key] = validation_report
+            except Exception as e:
+                report[key] = str(e)
+            return report
+
+        validate(self._conn.validate_query_params, ParamName.QUERY_PARAMS, query_params)
+        validate(self._conn.validate_path_params, ParamName.PATH_PARAMS, path_params)
+        if is_post_request(op_spec) or is_put_request(op_spec):
+            validate(self._conn.validate_data, ParamName.DATA, data)
+
+        if report:
+            raise ValidationError(report)
+
+    def is_upsert_operation_supported(self, op_name):
+        """
+        Checks if all operations required for upsert object operation are defined in 'operations'.
+
+        :param op_name: upsert operation name
+        :type op_name: str
+        :return: True if all criteria required to provide requested called operation are satisfied, otherwise False
+        :rtype: bool
+        """
+        model_name = _extract_model_from_upsert_operation(op_name)
+        operations = self.get_operation_specs_by_model_name(model_name)
+        return self._operation_checker.is_upsert_operation_supported(operations)
+
+    @staticmethod
+    def _get_operation_name(checker, operations):
+        for operation_name, op_spec in operations.items():
+            if checker(operation_name, op_spec):
+                return operation_name
+        raise FtdConfigurationError("Operation is not supported")
+
+    def _add_upserted_object(self, model_operations, params):
+        add_op_name = self._get_operation_name(self._operation_checker.is_add_operation, model_operations)
+        return self.add_object(add_op_name, params)
+
+    def _edit_upserted_object(self, model_operations, existing_object, params):
+        edit_op_name = self._get_operation_name(self._operation_checker.is_edit_operation, model_operations)
+        _set_default(params, 'path_params', {})
+        _set_default(params, 'data', {})
+
+        params['path_params']['objId'] = existing_object['id']
+        copy_identity_properties(existing_object, params['data'])
+        return self.edit_object(edit_op_name, params)
+
+    def upsert_object(self, op_name, params):
+        """
+        The wrapper on top of add object operation, get a list of objects and edit object operations that implement
+        upsert object operation. As a result, the object will be created if the object does not exist, if a single
+        object exists with requested 'params' this object will be updated otherwise, Exception will be raised.
+
+        :param op_name: upsert operation name
+        :type op_name: str
+        :param params: params that upsert operation should be executed with
+        :type params: dict
+        :return: upserted object representation
+        :rtype: dict
+        """
+        if not self.is_upsert_operation_supported(op_name):
+            raise FtdInvalidOperationNameError(op_name)
+
+        model_name = _extract_model_from_upsert_operation(op_name)
+        model_operations = self.get_operation_specs_by_model_name(model_name)
+
+        try:
+            return self._add_upserted_object(model_operations, params)
+        except FtdConfigurationError as e:
+            if e.obj:
+                return self._edit_upserted_object(model_operations, e.obj, params)
+            raise e
+
+
+def _set_default(params, field_name, value):
+    if field_name not in params or params[field_name] is None:
+        params[field_name] = value
+
+
+def is_post_request(operation_spec):
+    return operation_spec[OperationField.METHOD] == HTTPMethod.POST
+
+
+def is_put_request(operation_spec):
+    return operation_spec[OperationField.METHOD] == HTTPMethod.PUT
+
+
+def _extract_model_from_upsert_operation(op_name):
+    return op_name[len(OperationNamePrefix.UPSERT):]
+
+
+def _get_user_params(params):
+    return params.get(ParamName.DATA) or {}, params.get(ParamName.QUERY_PARAMS) or {}, params.get(
+        ParamName.PATH_PARAMS) or {}
+
+
+def iterate_over_pageable_resource(resource_func, params):
     """
     A generator function that iterates over a resource that supports pagination and lazily returns present items
     one by one.
 
-    :param resource_func: function that receives `query_params` named argument and returns a page of objects
+    :param resource_func: function that receives `params` argument and returns a page of objects
     :type resource_func: callable
-    :param query_params: initial dictionary of query parameters that will be passed to the resource_func
-    :type query_params: dict
+    :param params: initial dictionary of parameters that will be passed to the resource_func.
+                   Should contain `query_params` inside.
+    :type params: dict
     :return: an iterator containing returned items
     :rtype: iterator of dict
     """
-    query_params = {} if query_params is None else dict(query_params)
-    query_params.setdefault('limit', DEFAULT_PAGE_SIZE)
-    query_params.setdefault('offset', DEFAULT_OFFSET)
+    # creating a copy not to mutate passed dict
+    params = copy.deepcopy(params)
+    params[ParamName.QUERY_PARAMS].setdefault('limit', DEFAULT_PAGE_SIZE)
+    params[ParamName.QUERY_PARAMS].setdefault('offset', DEFAULT_OFFSET)
+    limit = int(params[ParamName.QUERY_PARAMS]['limit'])
+
+    def received_less_items_than_requested(items_in_response, items_expected):
+        if items_in_response == items_expected:
+            return False
+        elif items_in_response < items_expected:
+            return True
+
+        raise FtdUnexpectedResponse(
+            "Get List of Objects Response from the server contains more objects than requested. "
+            "There are {0} item(s) in the response while {1} was(ere) requested".format(items_in_response,
+                                                                                        items_expected)
+        )
+
+    while True:
+        result = resource_func(params=params)
 
-    result = resource_func(query_params=query_params)
-    while result['items']:
         for item in result['items']:
             yield item
+
+        if received_less_items_than_requested(len(result['items']), limit):
+            break
+
         # creating a copy not to mutate existing dict
-        query_params = dict(query_params)
-        query_params['offset'] = int(query_params['offset']) + int(query_params['limit'])
-        result = resource_func(query_params=query_params)
+        params = copy.deepcopy(params)
+        query_params = params[ParamName.QUERY_PARAMS]
+        query_params['offset'] = int(query_params['offset']) + limit

lib/ansible/module_utils/network/ftd/fdm_swagger_client.py

@@ -17,10 +17,11 @@
 #
 
 from ansible.module_utils.network.ftd.common import HTTPMethod
-from ansible.module_utils.six import integer_types, string_types
+from ansible.module_utils.six import integer_types, string_types, iteritems
 
 FILE_MODEL_NAME = '_File'
 SUCCESS_RESPONSE_CODE = '200'
+DELETE_PREFIX = 'delete'
 
 
 class OperationField:
@@ -28,12 +29,15 @@ class OperationField:
     METHOD = 'method'
     PARAMETERS = 'parameters'
     MODEL_NAME = 'modelName'
+    DESCRIPTION = 'description'
+    RETURN_MULTIPLE_ITEMS = 'returnMultipleItems'
 
 
 class SpecProp:
     DEFINITIONS = 'definitions'
     OPERATIONS = 'operations'
     MODELS = 'models'
+    MODEL_OPERATIONS = 'model_operations'
 
 
 class PropName:
@@ -51,6 +55,7 @@ class PropName:
     PROPERTIES = 'properties'
     RESPONSES = 'responses'
     NAME = 'name'
+    DESCRIPTION = 'description'
 
 
 class PropType:
@@ -68,6 +73,10 @@ class OperationParams:
     QUERY = 'query'
 
 
+class QueryParams:
+    FILTER = 'filter'
+
+
 def _get_model_name_from_url(schema_ref):
     path = schema_ref.split('/')
     return path[len(path) - 1]
@@ -89,13 +98,18 @@ class ValidationError(ValueError):
 
 class FdmSwaggerParser:
     _definitions = None
+    _base_path = None
 
-    def parse_spec(self, spec):
+    def parse_spec(self, spec, docs=None):
         """
-        This method simplifies a swagger format and also resolves a model name for each operation
-        :param spec: dict
-                    expect data in the swagger format see <https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md>
-        :rtype: (bool, string|dict)
+        This method simplifies a swagger format, resolves a model name for each operation, and adds documentation for
+        each operation and model if it is provided.
+
+        :param spec: An API specification in the swagger format, see <https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md>
+        :type spec: dict
+        :param spec: A documentation map containing descriptions for models, operations and operation parameters.
+        :type docs: dict
+        :rtype: dict
         :return:
         Ex.
             The models field contains model definition from swagger see <#https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md#definitions>
@@ -111,6 +125,7 @@ class FdmSwaggerParser:
                         'modelName': 'NetworkObject', # it is a link to the model from 'models'
                                                       # None - for a delete operation or we don't have information
                                                       # '_File' - if an endpoint works with files
+                        'returnMultipleItems': False, # shows if the operation returns a single item or an item list
                         'parameters': {
                             'path':{
                                 'param_name':{
@@ -129,26 +144,49 @@ class FdmSwaggerParser:
                         }
                     },
                     ...
+                },
+                'model_operations':{
+                    'model_name':{ # a list of operations available for the current model
+                        'operation_name':{
+                            ... # the same as in the operations section
+                        },
+                        ...
+                    },
+                    ...
                 }
             }
         """
         self._definitions = spec[SpecProp.DEFINITIONS]
-        config = {
+        self._base_path = spec[PropName.BASE_PATH]
+        operations = self._get_operations(spec)
+
+        if docs:
+            operations = self._enrich_operations_with_docs(operations, docs)
+            self._definitions = self._enrich_definitions_with_docs(self._definitions, docs)
+
+        return {
             SpecProp.MODELS: self._definitions,
-            SpecProp.OPERATIONS: self._get_operations(spec)
+            SpecProp.OPERATIONS: operations,
+            SpecProp.MODEL_OPERATIONS: self._get_model_operations(operations)
         }
-        return config
+
+    def _get_model_operations(self, operations):
+        model_operations = {}
+        for operations_name, params in iteritems(operations):
+            model_name = params[OperationField.MODEL_NAME]
+            model_operations.setdefault(model_name, {})[operations_name] = params
+        return model_operations
 
     def _get_operations(self, spec):
-        base_path = spec[PropName.BASE_PATH]
         paths_dict = spec[PropName.PATHS]
         operations_dict = {}
-        for url, operation_params in paths_dict.items():
-            for method, params in operation_params.items():
+        for url, operation_params in iteritems(paths_dict):
+            for method, params in iteritems(operation_params):
                 operation = {
                     OperationField.METHOD: method,
-                    OperationField.URL: base_path + url,
-                    OperationField.MODEL_NAME: self._get_model_name(method, params)
+                    OperationField.URL: self._base_path + url,
+                    OperationField.MODEL_NAME: self._get_model_name(method, params),
+                    OperationField.RETURN_MULTIPLE_ITEMS: self._return_multiple_items(params)
                 }
                 if OperationField.PARAMETERS in params:
                     operation[OperationField.PARAMETERS] = self._get_rest_params(params[OperationField.PARAMETERS])
@@ -157,14 +195,68 @@ class FdmSwaggerParser:
                 operations_dict[operation_id] = operation
         return operations_dict
 
+    def _enrich_operations_with_docs(self, operations, docs):
+        def get_operation_docs(op):
+            op_url = op[OperationField.URL][len(self._base_path):]
+            return docs[PropName.PATHS].get(op_url, {}).get(op[OperationField.METHOD], {})
+
+        for operation in operations.values():
+            operation_docs = get_operation_docs(operation)
+            operation[OperationField.DESCRIPTION] = operation_docs.get(PropName.DESCRIPTION, '')
+
+            if OperationField.PARAMETERS in operation:
+                param_descriptions = dict((p[PropName.NAME], p[PropName.DESCRIPTION])
+                                          for p in operation_docs.get(OperationField.PARAMETERS, {}))
+
+                for param_name, params_spec in operation[OperationField.PARAMETERS][OperationParams.PATH].items():
+                    params_spec[OperationField.DESCRIPTION] = param_descriptions.get(param_name, '')
+
+                for param_name, params_spec in operation[OperationField.PARAMETERS][OperationParams.QUERY].items():
+                    params_spec[OperationField.DESCRIPTION] = param_descriptions.get(param_name, '')
+
+        return operations
+
+    def _enrich_definitions_with_docs(self, definitions, docs):
+        for model_name, model_def in definitions.items():
+            model_docs = docs[SpecProp.DEFINITIONS].get(model_name, {})
+            model_def[PropName.DESCRIPTION] = model_docs.get(PropName.DESCRIPTION, '')
+            for prop_name, prop_spec in model_def.get(PropName.PROPERTIES, {}).items():
+                prop_spec[PropName.DESCRIPTION] = model_docs.get(PropName.PROPERTIES, {}).get(prop_name, '')
+                prop_spec[PropName.REQUIRED] = prop_name in model_def.get(PropName.REQUIRED, [])
+        return definitions
+
     def _get_model_name(self, method, params):
         if method == HTTPMethod.GET:
             return self._get_model_name_from_responses(params)
         elif method == HTTPMethod.POST or method == HTTPMethod.PUT:
             return self._get_model_name_for_post_put_requests(params)
+        elif method == HTTPMethod.DELETE:
+            return self._get_model_name_from_delete_operation(params)
         else:
             return None
 
+    @staticmethod
+    def _return_multiple_items(op_params):
+        """
+        Defines if the operation returns one item or a list of items.
+
+        :param op_params: operation specification
+        :return: True if the operation returns a list of items, otherwise False
+        """
+        try:
+            schema = op_params[PropName.RESPONSES][SUCCESS_RESPONSE_CODE][PropName.SCHEMA]
+            return PropName.ITEMS in schema[PropName.PROPERTIES]
+        except KeyError:
+            return False
+
+    def _get_model_name_from_delete_operation(self, params):
+        operation_id = params[PropName.OPERATION_ID]
+        if operation_id.startswith(DELETE_PREFIX):
+            model_name = operation_id[len(DELETE_PREFIX):]
+            if model_name in self._definitions:
+                return model_name
+        return None
+
     def _get_model_name_for_post_put_requests(self, params):
         model_name = None
         if OperationField.PARAMETERS in params:
@@ -429,7 +521,8 @@ class FdmSwaggerValidator:
             self._add_invalid_type_report(status, path, '', PropType.OBJECT, data)
             return None
 
-        self._check_required_fields(status, model[PropName.REQUIRED], data, path)
+        if PropName.REQUIRED in model:
+            self._check_required_fields(status, model[PropName.REQUIRED], data, path)
 
         model_properties = model[PropName.PROPERTIES]
         for prop in model_properties.keys():
@@ -472,14 +565,25 @@ class FdmSwaggerValidator:
 
     @staticmethod
     def _is_correct_simple_types(expected_type, value):
+        def is_numeric_string(s):
+            try:
+                float(s)
+                return True
+            except ValueError:
+                return False
+
         if expected_type == PropType.STRING:
             return isinstance(value, string_types)
         elif expected_type == PropType.BOOLEAN:
             return isinstance(value, bool)
         elif expected_type == PropType.INTEGER:
-            return isinstance(value, integer_types) and not isinstance(value, bool)
+            is_integer = isinstance(value, integer_types) and not isinstance(value, bool)
+            is_digit_string = isinstance(value, string_types) and value.isdigit()
+            return is_integer or is_digit_string
         elif expected_type == PropType.NUMBER:
-            return isinstance(value, (integer_types, float)) and not isinstance(value, bool)
+            is_number = isinstance(value, (integer_types, float)) and not isinstance(value, bool)
+            is_numeric_string = isinstance(value, string_types) and is_numeric_string(value)
+            return is_number or is_numeric_string
         return False
 
     @staticmethod