New modules and updated HTTP API plugin for FTD devices (#44578)

* Add common and Swagger client utils for FTD modules

* Update FTD HTTP API plugin and add unit tests for it

* Add configuration layer handling object idempotency

* Add ftd_configuration module with unit tests

* Add ftd_file_download and ftd_file_upload modules with unit tests

* Validate operation data and parameters

* Fix ansible-doc, boilerplate and import errors

* Fix pip8 sanity errors

* Update object comparison to work recursively

* Add copyright

lib/ansible/modules/network/ftd/ftd_configuration.py

@@ -0,0 +1,219 @@
+#!/usr/bin/python
+
+# Copyright (c) 2018 Cisco and/or its affiliates.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'network'}
+
+DOCUMENTATION = """
+---
+module: ftd_configuration
+short_description: Manages configuration on Cisco FTD devices over REST API
+description:
+  - Manages configuration on Cisco FTD devices including creating, updating, removing configuration objects,
+    scheduling and staring jobs, deploying pending changes, etc. All operation are performed over REST API.
+version_added: "2.7"
+author: "Cisco Systems, Inc."
+options:
+  operation:
+    description:
+      - The name of the operation to execute. Commonly, the operation starts with 'add', 'edit', 'get'
+       or 'delete' verbs, but can have an arbitrary name too.
+    required: true
+  data:
+    description:
+      - Key-value pairs that should be sent as body parameters in a REST API call
+  query_params:
+    description:
+      - Key-value pairs that should be sent as query parameters in a REST API call.
+  path_params:
+    description:
+      - Key-value pairs that should be sent as path parameters in a REST API call.
+  register_as:
+    description:
+      - Specifies Ansible fact name that is used to register received response from the FTD device.
+  filters:
+    description:
+      - Key-value dict that represents equality filters. Every key is a property name and value is its desired value.
+        If multiple filters are present, they are combined with logical operator AND.
+"""
+
+EXAMPLES = """
+- name: Create a network object
+  ftd_configuration:
+    operation: "addNetworkObject"
+    data:
+      name: "Ansible-network-host"
+      description: "From Ansible with love"
+      subType: "HOST"
+      value: "192.168.2.0"
+      dnsResolution: "IPV4_AND_IPV6"
+      type: "networkobject"
+      isSystemDefined: false
+    register_as: "hostNetwork"
+
+- name: Delete the network object
+  ftd_configuration:
+    operation: "deleteNetworkObject"
+    path_params:
+      objId: "{{ hostNetwork['id'] }}"
+"""
+
+RETURN = """
+response:
+  description: HTTP response returned from the API call.
+  returned: success
+  type: dict
+"""
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.connection import Connection
+from ansible.module_utils.network.ftd.common import HTTPMethod, construct_ansible_facts, FtdConfigurationError, \
+    FtdServerError
+from ansible.module_utils.network.ftd.configuration import BaseConfigurationResource
+from ansible.module_utils.network.ftd.fdm_swagger_client import OperationField, ValidationError
+
+
+def is_post_request(operation_spec):
+    return operation_spec[OperationField.METHOD] == HTTPMethod.POST
+
+
+def is_put_request(operation_spec):
+    return operation_spec[OperationField.METHOD] == HTTPMethod.PUT
+
+
+def is_add_operation(operation_name, operation_spec):
+    # Some endpoints have non-CRUD operations, so checking operation name is required in addition to the HTTP method
+    return operation_name.startswith('add') and is_post_request(operation_spec)
+
+
+def is_edit_operation(operation_name, operation_spec):
+    # Some endpoints have non-CRUD operations, so checking operation name is required in addition to the HTTP method
+    return operation_name.startswith('edit') and is_put_request(operation_spec)
+
+
+def is_delete_operation(operation_name, operation_spec):
+    # Some endpoints have non-CRUD operations, so checking operation name is required in addition to the HTTP method
+    return operation_name.startswith('delete') and operation_spec[OperationField.METHOD] == HTTPMethod.DELETE
+
+
+def validate_params(connection, op_name, query_params, path_params, data, op_spec):
+    report = {}
+
+    def validate(validation_method, field_name, params):
+        key = 'Invalid %s provided' % field_name
+        try:
+            is_valid, validation_report = validation_method(op_name, params)
+            if not is_valid:
+                report[key] = validation_report
+        except Exception as e:
+            report[key] = str(e)
+        return report
+
+    validate(connection.validate_query_params, 'query_params', query_params)
+    validate(connection.validate_path_params, 'path_params', path_params)
+    if is_post_request(op_spec) or is_post_request(op_spec):
+        validate(connection.validate_data, 'data', data)
+
+    if report:
+        raise ValidationError(report)
+
+
+def is_find_by_filter_operation(operation_name, operation_spec, params):
+    """
+    Checks whether the called operation is 'find by filter'. This operation fetches all objects and finds
+    the matching ones by the given filter. As filtering is done on the client side, this operation should be used
+    only when selected filters are not implemented on the server side.
+
+    :param operation_name: name of the operation being called by the user
+    :type operation_name: str
+    :param operation_spec: specification of the operation being called by the user
+    :type operation_spec: dict
+    :param params: module parameters
+    :return: True if called operation is find by filter, otherwise False
+    :rtype: bool
+    """
+    is_get_list_operation = operation_name.startswith('get') and operation_name.endswith('List')
+    is_get_method = operation_spec[OperationField.METHOD] == HTTPMethod.GET
+    return is_get_list_operation and is_get_method and params['filters']
+
+
+def main():
+    fields = dict(
+        operation=dict(type='str', required=True),
+        data=dict(type='dict'),
+        query_params=dict(type='dict'),
+        path_params=dict(type='dict'),
+        register_as=dict(type='str'),
+        filters=dict(type='dict')
+    )
+    module = AnsibleModule(argument_spec=fields,
+                           supports_check_mode=True)
+    params = module.params
+
+    connection = Connection(module._socket_path)
+
+    op_name = params['operation']
+    op_spec = connection.get_operation_spec(op_name)
+    if op_spec is None:
+        module.fail_json(msg='Invalid operation name provided: %s' % op_name)
+
+    data, query_params, path_params = params['data'], params['query_params'], params['path_params']
+
+    try:
+        validate_params(connection, op_name, query_params, path_params, data, op_spec)
+    except ValidationError as e:
+        module.fail_json(msg=e.args[0])
+
+    try:
+        if module.check_mode:
+            module.exit_json(changed=False)
+
+        resource = BaseConfigurationResource(connection)
+        url = op_spec[OperationField.URL]
+
+        if is_add_operation(op_name, op_spec):
+            resp = resource.add_object(url, data, path_params, query_params)
+        elif is_edit_operation(op_name, op_spec):
+            resp = resource.edit_object(url, data, path_params, query_params)
+        elif is_delete_operation(op_name, op_spec):
+            resp = resource.delete_object(url, path_params)
+        elif is_find_by_filter_operation(op_name, op_spec, params):
+            resp = resource.get_objects_by_filter(url, params['filters'], path_params,
+                                                  query_params)
+        else:
+            resp = resource.send_request(url, op_spec[OperationField.METHOD], data,
+                                         path_params,
+                                         query_params)
+
+        module.exit_json(changed=resource.config_changed, response=resp,
+                         ansible_facts=construct_ansible_facts(resp, module.params))
+    except FtdConfigurationError as e:
+        module.fail_json(msg='Failed to execute %s operation because of the configuration error: %s' % (op_name, e))
+    except FtdServerError as e:
+        module.fail_json(msg='Server returned an error trying to execute %s operation. Status code: %s. '
+                             'Server response: %s' % (op_name, e.code, e.response))
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/modules/network/ftd/ftd_file_download.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python
+
+# Copyright (c) 2018 Cisco and/or its affiliates.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'network'}
+
+DOCUMENTATION = """
+---
+module: ftd_file_download
+short_description: Downloads files from Cisco FTD devices over HTTP(S)
+description:
+  - Downloads files from Cisco FTD devices including pending changes, disk files, certificates,
+    troubleshoot reports, and backups.
+version_added: "2.7"
+author: "Cisco Systems, Inc."
+options:
+  operation:
+    description:
+      - The name of the operation to execute.
+      - Only operations that return a file can be used in this module.
+    required: true
+  path_params:
+    description:
+      - Key-value pairs that should be sent as path parameters in a REST API call.
+  destination:
+    description:
+      - Absolute path of where to download the file to.
+      - If destination is a directory, the module uses a filename from 'Content-Disposition' header specified by the server.
+    required: true
+"""
+
+EXAMPLES = """
+- name: Download pending changes
+  ftd_file_download:
+    operation: 'getdownload'
+    path_params:
+      objId: 'default'
+    destination: /tmp/
+"""
+
+RETURN = """
+msg:
+    description: the error message describing why the module failed
+    returned: error
+    type: string
+"""
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.connection import Connection
+from ansible.module_utils.network.ftd.common import FtdServerError, HTTPMethod
+from ansible.module_utils.network.ftd.fdm_swagger_client import OperationField, ValidationError, FILE_MODEL_NAME
+
+
+def is_download_operation(op_spec):
+    return op_spec[OperationField.METHOD] == HTTPMethod.GET and op_spec[OperationField.MODEL_NAME] == FILE_MODEL_NAME
+
+
+def validate_params(connection, op_name, path_params):
+    field_name = 'Invalid path_params provided'
+    try:
+        is_valid, validation_report = connection.validate_path_params(op_name, path_params)
+        if not is_valid:
+            raise ValidationError({
+                field_name: validation_report
+            })
+    except Exception as e:
+        raise ValidationError({
+            field_name: str(e)
+        })
+
+
+def main():
+    fields = dict(
+        operation=dict(type='str', required=True),
+        path_params=dict(type='dict'),
+        destination=dict(type='path', required=True)
+    )
+    module = AnsibleModule(argument_spec=fields,
+                           supports_check_mode=True)
+    params = module.params
+    connection = Connection(module._socket_path)
+
+    op_name = params['operation']
+    op_spec = connection.get_operation_spec(op_name)
+    if op_spec is None:
+        module.fail_json(msg='Operation with specified name is not found: %s' % op_name)
+    if not is_download_operation(op_spec):
+        module.fail_json(
+            msg='Invalid download operation: %s. The operation must make GET request and return a file.' %
+                op_name)
+
+    try:
+        path_params = params['path_params']
+        validate_params(connection, op_name, path_params)
+        if module.check_mode:
+            module.exit_json(changed=False)
+        connection.download_file(op_spec[OperationField.URL], params['destination'], path_params)
+        module.exit_json(changed=False)
+    except FtdServerError as e:
+        module.fail_json(msg='Download request for %s operation failed. Status code: %s. '
+                             'Server response: %s' % (op_name, e.code, e.response))
+    except ValidationError as e:
+        module.fail_json(msg=e.args[0])
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/modules/network/ftd/ftd_file_upload.py

@@ -0,0 +1,105 @@
+#!/usr/bin/python
+
+# Copyright (c) 2018 Cisco and/or its affiliates.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'network'}
+
+DOCUMENTATION = """
+---
+module: ftd_file_upload
+short_description: Uploads files to Cisco FTD devices over HTTP(S)
+description:
+  - Uploads files to Cisco FTD devices including disk files, backups, and upgrades.
+version_added: "2.7"
+author: "Cisco Systems, Inc."
+options:
+  operation:
+    description:
+      - The name of the operation to execute.
+      - Only operations that upload file can be used in this module.
+    required: true
+  fileToUpload:
+    description:
+      - Absolute path to the file that should be uploaded.
+    required: true
+  register_as:
+    description:
+      - Specifies Ansible fact name that is used to register received response from the FTD device.
+"""
+
+EXAMPLES = """
+- name: Upload disk file
+  ftd_file_upload:
+    operation: 'postuploaddiskfile'
+    fileToUpload: /tmp/test1.txt
+"""
+
+RETURN = """
+msg:
+    description: the error message describing why the module failed
+    returned: error
+    type: string
+"""
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.connection import Connection
+from ansible.module_utils.network.ftd.common import construct_ansible_facts, FtdServerError, HTTPMethod
+from ansible.module_utils.network.ftd.fdm_swagger_client import OperationField
+
+
+def is_upload_operation(op_spec):
+    return op_spec[OperationField.METHOD] == HTTPMethod.POST or 'UploadStatus' in op_spec[OperationField.MODEL_NAME]
+
+
+def main():
+    fields = dict(
+        operation=dict(type='str', required=True),
+        fileToUpload=dict(type='path', required=True),
+        register_as=dict(type='str'),
+    )
+    module = AnsibleModule(argument_spec=fields,
+                           supports_check_mode=True)
+    params = module.params
+    connection = Connection(module._socket_path)
+
+    op_spec = connection.get_operation_spec(params['operation'])
+    if op_spec is None:
+        module.fail_json(msg='Operation with specified name is not found: %s' % params['operation'])
+    if not is_upload_operation(op_spec):
+        module.fail_json(
+            msg='Invalid upload operation: %s. The operation must make POST request and return UploadStatus model.' %
+                params['operation'])
+
+    try:
+        if module.check_mode:
+            module.exit_json()
+        resp = connection.upload_file(params['fileToUpload'], op_spec[OperationField.URL])
+        module.exit_json(changed=True, response=resp, ansible_facts=construct_ansible_facts(resp, module.params))
+    except FtdServerError as e:
+        module.fail_json(msg='Upload request for %s operation failed. Status code: %s. '
+                             'Server response: %s' % (params['operation'], e.code, e.response))
+
+
+if __name__ == '__main__':
+    main()