Initial commit

plugins/httpapi/exos.py

@@ -0,0 +1,252 @@
+# Copyright (c) 2019 Extreme Networks.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+author:
+  - "Ujwal Komarla (@ujwalkomarla)"
+httpapi: exos
+short_description: Use EXOS REST APIs to communicate with EXOS platform
+description:
+  - This plugin provides low level abstraction api's to send REST API
+    requests to EXOS network devices and receive JSON responses.
+'''
+
+import json
+import re
+from ansible.module_utils._text import to_text
+from ansible.module_utils.connection import ConnectionError
+from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.utils import to_list
+from ansible.plugins.httpapi import HttpApiBase
+import ansible.module_utils.six.moves.http_cookiejar as cookiejar
+from ansible.module_utils.common._collections_compat import Mapping
+from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.config import NetworkConfig, dumps
+
+
+class HttpApi(HttpApiBase):
+
+    def __init__(self, *args, **kwargs):
+        super(HttpApi, self).__init__(*args, **kwargs)
+        self._device_info = None
+        self._auth_token = cookiejar.CookieJar()
+
+    def login(self, username, password):
+        auth_path = '/auth/token'
+        credentials = {'username': username, 'password': password}
+        self.send_request(path=auth_path, data=json.dumps(credentials), method='POST')
+
+    def logout(self):
+        pass
+
+    def handle_httperror(self, exc):
+        return False
+
+    def send_request(self, path, data=None, method='GET', **message_kwargs):
+        headers = {'Content-Type': 'application/json'}
+        response, response_data = self.connection.send(path, data, method=method, cookies=self._auth_token, headers=headers, **message_kwargs)
+        try:
+            if response.status == 204:
+                response_data = {}
+            else:
+                response_data = json.loads(to_text(response_data.getvalue()))
+        except ValueError:
+            raise ConnectionError('Response was not valid JSON, got {0}'.format(
+                to_text(response_data.getvalue())
+            ))
+        return response_data
+
+    def run_commands(self, commands, check_rc=True):
+        if commands is None:
+            raise ValueError("'commands' value is required")
+
+        headers = {'Content-Type': 'application/json'}
+        responses = list()
+        for cmd in to_list(commands):
+            if not isinstance(cmd, Mapping):
+                cmd = {'command': cmd}
+
+            cmd['command'] = strip_run_script_cli2json(cmd['command'])
+
+            output = cmd.pop('output', None)
+            if output and output not in self.get_option_values().get('output'):
+                raise ValueError("'output' value is %s is invalid. Valid values are %s" % (output, ','.join(self.get_option_values().get('output'))))
+
+            data = request_builder(cmd['command'])
+
+            response, response_data = self.connection.send('/jsonrpc', data, cookies=self._auth_token, headers=headers, method='POST')
+            try:
+                response_data = json.loads(to_text(response_data.getvalue()))
+            except ValueError:
+                raise ConnectionError('Response was not valid JSON, got {0}'.format(
+                    to_text(response_data.getvalue())
+                ))
+
+            if response_data.get('error', None):
+                raise ConnectionError("Request Error, got {0}".format(response_data['error']))
+            if not response_data.get('result', None):
+                raise ConnectionError("Request Error, got {0}".format(response_data))
+
+            response_data = response_data['result']
+
+            if output and output == 'text':
+                statusOut = getKeyInResponse(response_data, 'status')
+                cliOut = getKeyInResponse(response_data, 'CLIoutput')
+                if statusOut == "ERROR":
+                    raise ConnectionError("Command error({1}) for request {0}".format(cmd['command'], cliOut))
+                if cliOut is None:
+                    raise ValueError("Response for request {0} doesn't have the CLIoutput field, got {1}".format(cmd['command'], response_data))
+                response_data = cliOut
+
+            responses.append(response_data)
+        return responses
+
+    def get_device_info(self):
+        device_info = {}
+        device_info['network_os'] = 'exos'
+
+        reply = self.run_commands({'command': 'show switch detail', 'output': 'text'})
+        data = to_text(reply, errors='surrogate_or_strict').strip()
+
+        match = re.search(r'ExtremeXOS version  (\S+)', data)
+        if match:
+            device_info['network_os_version'] = match.group(1)
+
+        match = re.search(r'System Type: +(\S+)', data)
+        if match:
+            device_info['network_os_model'] = match.group(1)
+
+        match = re.search(r'SysName: +(\S+)', data)
+        if match:
+            device_info['network_os_hostname'] = match.group(1)
+
+        return device_info
+
+    def get_device_operations(self):
+        return {
+            'supports_diff_replace': False,         # identify if config should be merged or replaced is supported
+            'supports_commit': False,               # identify if commit is supported by device or not
+            'supports_rollback': False,             # identify if rollback is supported or not
+            'supports_defaults': True,              # identify if fetching running config with default is supported
+            'supports_commit_comment': False,       # identify if adding comment to commit is supported of not
+            'supports_onbox_diff': False,           # identify if on box diff capability is supported or not
+            'supports_generate_diff': True,         # identify if diff capability is supported within plugin
+            'supports_multiline_delimiter': False,  # identify if multiline demiliter is supported within config
+            'supports_diff_match': True,            # identify if match is supported
+            'supports_diff_ignore_lines': True,     # identify if ignore line in diff is supported
+            'supports_config_replace': False,       # identify if running config replace with candidate config is supported
+            'supports_admin': False,                # identify if admin configure mode is supported or not
+            'supports_commit_label': False          # identify if commit label is supported or not
+        }
+
+    def get_option_values(self):
+        return {
+            'format': ['text', 'json'],
+            'diff_match': ['line', 'strict', 'exact', 'none'],
+            'diff_replace': ['line', 'block'],
+            'output': ['text', 'json']
+        }
+
+    def get_capabilities(self):
+        result = {}
+        result['rpc'] = ['get_default_flag', 'run_commands', 'get_config', 'send_request', 'get_capabilities', 'get_diff']
+        result['device_info'] = self.get_device_info()
+        result['device_operations'] = self.get_device_operations()
+        result.update(self.get_option_values())
+        result['network_api'] = 'exosapi'
+        return json.dumps(result)
+
+    def get_default_flag(self):
+        # The flag to modify the command to collect configuration with defaults
+        return 'detail'
+
+    def get_diff(self, candidate=None, running=None, diff_match='line', diff_ignore_lines=None, path=None, diff_replace='line'):
+        diff = {}
+        device_operations = self.get_device_operations()
+        option_values = self.get_option_values()
+
+        if candidate is None and device_operations['supports_generate_diff']:
+            raise ValueError("candidate configuration is required to generate diff")
+
+        if diff_match not in option_values['diff_match']:
+            raise ValueError("'match' value %s in invalid, valid values are %s" % (diff_match, ', '.join(option_values['diff_match'])))
+
+        if diff_replace not in option_values['diff_replace']:
+            raise ValueError("'replace' value %s in invalid, valid values are %s" % (diff_replace, ', '.join(option_values['diff_replace'])))
+
+        # prepare candidate configuration
+        candidate_obj = NetworkConfig(indent=1)
+        candidate_obj.load(candidate)
+
+        if running and diff_match != 'none' and diff_replace != 'config':
+            # running configuration
+            running_obj = NetworkConfig(indent=1, contents=running, ignore_lines=diff_ignore_lines)
+            configdiffobjs = candidate_obj.difference(running_obj, path=path, match=diff_match, replace=diff_replace)
+
+        else:
+            configdiffobjs = candidate_obj.items
+
+        diff['config_diff'] = dumps(configdiffobjs, 'commands') if configdiffobjs else ''
+        return diff
+
+    def get_config(self, source='running', format='text', flags=None):
+        options_values = self.get_option_values()
+        if format not in options_values['format']:
+            raise ValueError("'format' value %s is invalid. Valid values are %s" % (format, ','.join(options_values['format'])))
+
+        lookup = {'running': 'show configuration', 'startup': 'debug cfgmgr show configuration file'}
+        if source not in lookup:
+            raise ValueError("fetching configuration from %s is not supported" % source)
+
+        cmd = {'command': lookup[source], 'output': 'text'}
+
+        if source == 'startup':
+            reply = self.run_commands({'command': 'show switch', 'format': 'text'})
+            data = to_text(reply, errors='surrogate_or_strict').strip()
+            match = re.search(r'Config Selected: +(\S+)\.cfg', data, re.MULTILINE)
+            if match:
+                cmd['command'] += match.group(1)
+            else:
+                # No Startup(/Selected) Config
+                return {}
+
+        cmd['command'] += ' '.join(to_list(flags))
+        cmd['command'] = cmd['command'].strip()
+
+        return self.run_commands(cmd)[0]
+
+
+def request_builder(command, reqid=""):
+    return json.dumps(dict(jsonrpc='2.0', id=reqid, method='cli', params=to_list(command)))
+
+
+def strip_run_script_cli2json(command):
+    if to_text(command, errors="surrogate_then_replace").startswith('run script cli2json.py'):
+        command = str(command).replace('run script cli2json.py', '')
+    return command
+
+
+def getKeyInResponse(response, key):
+    keyOut = None
+    for item in response:
+        if key in item:
+            keyOut = item[key]
+            break
+    return keyOut

plugins/httpapi/fortianalyzer.py

@@ -0,0 +1,453 @@
+# Copyright (c) 2018 Fortinet and/or its affiliates.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+author:
+    - Luke Weighall (@lweighall)
+    - Andrew Welsh (@Ghilli3)
+    - Jim Huber (@p4r4n0y1ng)
+httpapi : fortianalyzer
+short_description: HttpApi Plugin for Fortinet FortiAnalyzer Appliance or VM.
+description:
+  - This HttpApi plugin provides methods to connect to Fortinet FortiAnalyzer Appliance or VM via JSON RPC API.
+
+'''
+
+import json
+from ansible.plugins.httpapi import HttpApiBase
+from ansible.module_utils.basic import to_text
+from ansible_collections.community.general.plugins.module_utils.network.fortianalyzer.common import BASE_HEADERS
+from ansible_collections.community.general.plugins.module_utils.network.fortianalyzer.common import FAZBaseException
+from ansible_collections.community.general.plugins.module_utils.network.fortianalyzer.common import FAZCommon
+from ansible_collections.community.general.plugins.module_utils.network.fortianalyzer.common import FAZMethods
+
+
+class HttpApi(HttpApiBase):
+    def __init__(self, connection):
+        super(HttpApi, self).__init__(connection)
+        self._req_id = 0
+        self._sid = None
+        self._url = "/jsonrpc"
+        self._host = None
+        self._tools = FAZCommon
+        self._debug = False
+        self._connected_faz = None
+        self._last_response_msg = None
+        self._last_response_code = None
+        self._last_data_payload = None
+        self._last_url = None
+        self._last_response_raw = None
+        self._locked_adom_list = list()
+        self._locked_adoms_by_user = list()
+        self._uses_workspace = False
+        self._uses_adoms = False
+        self._adom_list = list()
+        self._logged_in_user = None
+
+    def set_become(self, become_context):
+        """
+        ELEVATION IS NOT REQUIRED ON FORTINET DEVICES - SKIPPED
+        :param become_context: Unused input.
+        :return: None
+        """
+        return None
+
+    def update_auth(self, response, response_data):
+        """
+        TOKENS ARE NOT USED SO NO NEED TO UPDATE AUTH
+        :param response: Unused input.
+        :param response_data Unused_input.
+        :return: None
+        """
+        return None
+
+    def login(self, username, password):
+        """
+        This function will log the plugin into FortiAnalyzer, and return the results.
+        :param username: Username of FortiAnalyzer Admin
+        :param password: Password of FortiAnalyzer Admin
+
+        :return: Dictionary of status if it logged in or not.
+        """
+
+        self._logged_in_user = username
+        self.send_request(FAZMethods.EXEC, self._tools.format_request(FAZMethods.EXEC, "sys/login/user",
+                                                                      passwd=password, user=username,))
+
+        if "FortiAnalyzer object connected to FortiAnalyzer" in self.__str__():
+            # If Login worked then inspect the FortiAnalyzer for Workspace Mode, and it's system information.
+            self.inspect_faz()
+            return
+        else:
+            raise FAZBaseException(msg="Unknown error while logging in...connection was lost during login operation..."
+                                       " Exiting")
+
+    def inspect_faz(self):
+        # CHECK FOR WORKSPACE MODE TO SEE IF WE HAVE TO ENABLE ADOM LOCKS
+        status = self.get_system_status()
+        if status[0] == -11:
+            # THE CONNECTION GOT LOST SOMEHOW, REMOVE THE SID AND REPORT BAD LOGIN
+            self.logout()
+            raise FAZBaseException(msg="Error -11 -- the Session ID was likely malformed somehow. Contact authors."
+                                       " Exiting")
+        elif status[0] == 0:
+            try:
+                self.check_mode()
+                if self._uses_adoms:
+                    self.get_adom_list()
+                if self._uses_workspace:
+                    self.get_locked_adom_list()
+                self._connected_faz = status[1]
+                self._host = self._connected_faz["Hostname"]
+            except Exception:
+                pass
+        return
+
+    def logout(self):
+        """
+        This function will logout of the FortiAnalyzer.
+        """
+        if self.sid is not None:
+            # IF WE WERE USING WORKSPACES, THEN CLEAN UP OUR LOCKS IF THEY STILL EXIST
+            if self.uses_workspace:
+                self.get_lock_info()
+                self.run_unlock()
+            ret_code, response = self.send_request(FAZMethods.EXEC,
+                                                   self._tools.format_request(FAZMethods.EXEC, "sys/logout"))
+            self.sid = None
+            return ret_code, response
+
+    def send_request(self, method, params):
+        """
+        Responsible for actual sending of data to the connection httpapi base plugin. Does some formatting as well.
+        :param params: A formatted dictionary that was returned by self.common_datagram_params()
+        before being called here.
+        :param method: The preferred API Request method (GET, ADD, POST, etc....)
+        :type method: basestring
+
+        :return: Dictionary of status if it logged in or not.
+        """
+
+        try:
+            if self.sid is None and params[0]["url"] != "sys/login/user":
+                try:
+                    self.connection._connect()
+                except Exception as err:
+                    raise FAZBaseException(
+                        msg="An problem happened with the httpapi plugin self-init connection process. "
+                            "Error: " + to_text(err))
+        except IndexError:
+            raise FAZBaseException("An attempt was made at communicating with a FAZ with "
+                                   "no valid session and an incorrectly formatted request.")
+        except Exception:
+            raise FAZBaseException("An attempt was made at communicating with a FAZ with "
+                                   "no valid session and an unexpected error was discovered.")
+
+        self._update_request_id()
+        json_request = {
+            "method": method,
+            "params": params,
+            "session": self.sid,
+            "id": self.req_id,
+            "verbose": 1
+        }
+        data = json.dumps(json_request, ensure_ascii=False).replace('\\\\', '\\')
+        try:
+            # Sending URL and Data in Unicode, per Ansible Specifications for Connection Plugins
+            response, response_data = self.connection.send(path=to_text(self._url), data=to_text(data),
+                                                           headers=BASE_HEADERS)
+            # Get Unicode Response - Must convert from StringIO to unicode first so we can do a replace function below
+            result = json.loads(to_text(response_data.getvalue()))
+            self._update_self_from_response(result, self._url, data)
+            return self._handle_response(result)
+        except Exception as err:
+            raise FAZBaseException(err)
+
+    def _handle_response(self, response):
+        self._set_sid(response)
+        if isinstance(response["result"], list):
+            result = response["result"][0]
+        else:
+            result = response["result"]
+        if "data" in result:
+            return result["status"]["code"], result["data"]
+        else:
+            return result["status"]["code"], result
+
+    def _update_self_from_response(self, response, url, data):
+        self._last_response_raw = response
+        if isinstance(response["result"], list):
+            result = response["result"][0]
+        else:
+            result = response["result"]
+        if "status" in result:
+            self._last_response_code = result["status"]["code"]
+            self._last_response_msg = result["status"]["message"]
+            self._last_url = url
+            self._last_data_payload = data
+
+    def _set_sid(self, response):
+        if self.sid is None and "session" in response:
+            self.sid = response["session"]
+
+    def return_connected_faz(self):
+        """
+        Returns the data stored under self._connected_faz
+
+        :return: dict
+        """
+        try:
+            if self._connected_faz:
+                return self._connected_faz
+        except Exception:
+            raise FAZBaseException("Couldn't Retrieve Connected FAZ Stats")
+
+    def get_system_status(self):
+        """
+        Returns the system status page from the FortiAnalyzer, for logging and other uses.
+        return: status
+        """
+        status = self.send_request(FAZMethods.GET, self._tools.format_request(FAZMethods.GET, "sys/status"))
+        return status
+
+    @property
+    def debug(self):
+        return self._debug
+
+    @debug.setter
+    def debug(self, val):
+        self._debug = val
+
+    @property
+    def req_id(self):
+        return self._req_id
+
+    @req_id.setter
+    def req_id(self, val):
+        self._req_id = val
+
+    def _update_request_id(self, reqid=0):
+        self.req_id = reqid if reqid != 0 else self.req_id + 1
+
+    @property
+    def sid(self):
+        return self._sid
+
+    @sid.setter
+    def sid(self, val):
+        self._sid = val
+
+    def __str__(self):
+        if self.sid is not None and self.connection._url is not None:
+            return "FortiAnalyzer object connected to FortiAnalyzer: " + to_text(self.connection._url)
+        return "FortiAnalyzer object with no valid connection to a FortiAnalyzer appliance."
+
+    ##################################
+    # BEGIN DATABASE LOCK CONTEXT CODE
+    ##################################
+
+    @property
+    def uses_workspace(self):
+        return self._uses_workspace
+
+    @uses_workspace.setter
+    def uses_workspace(self, val):
+        self._uses_workspace = val
+
+    @property
+    def uses_adoms(self):
+        return self._uses_adoms
+
+    @uses_adoms.setter
+    def uses_adoms(self, val):
+        self._uses_adoms = val
+
+    def add_adom_to_lock_list(self, adom):
+        if adom not in self._locked_adom_list:
+            self._locked_adom_list.append(adom)
+
+    def remove_adom_from_lock_list(self, adom):
+        if adom in self._locked_adom_list:
+            self._locked_adom_list.remove(adom)
+
+    def check_mode(self):
+        """
+        Checks FortiAnalyzer for the use of Workspace mode
+        """
+        url = "/cli/global/system/global"
+        code, resp_obj = self.send_request(FAZMethods.GET,
+                                           self._tools.format_request(FAZMethods.GET,
+                                                                      url,
+                                                                      fields=["workspace-mode", "adom-status"]))
+        try:
+            if resp_obj["workspace-mode"] == "workflow":
+                self.uses_workspace = True
+            elif resp_obj["workspace-mode"] == "disabled":
+                self.uses_workspace = False
+        except KeyError:
+            self.uses_workspace = False
+        except Exception:
+            raise FAZBaseException(msg="Couldn't determine workspace-mode in the plugin")
+        try:
+            if resp_obj["adom-status"] in [1, "enable"]:
+                self.uses_adoms = True
+            else:
+                self.uses_adoms = False
+        except KeyError:
+            self.uses_adoms = False
+        except Exception:
+            raise FAZBaseException(msg="Couldn't determine adom-status in the plugin")
+
+    def run_unlock(self):
+        """
+        Checks for ADOM status, if locked, it will unlock
+        """
+        for adom_locked in self._locked_adoms_by_user:
+            adom = adom_locked["adom"]
+            self.unlock_adom(adom)
+
+    def lock_adom(self, adom=None, *args, **kwargs):
+        """
+        Locks an ADOM for changes
+        """
+        if adom:
+            if adom.lower() == "global":
+                url = "/dvmdb/global/workspace/lock/"
+            else:
+                url = "/dvmdb/adom/{adom}/workspace/lock/".format(adom=adom)
+        else:
+            url = "/dvmdb/adom/root/workspace/lock"
+        code, respobj = self.send_request(FAZMethods.EXEC, self._tools.format_request(FAZMethods.EXEC, url))
+        if code == 0 and respobj["status"]["message"].lower() == "ok":
+            self.add_adom_to_lock_list(adom)
+        return code, respobj
+
+    def unlock_adom(self, adom=None, *args, **kwargs):
+        """
+        Unlocks an ADOM after changes
+        """
+        if adom:
+            if adom.lower() == "global":
+                url = "/dvmdb/global/workspace/unlock/"
+            else:
+                url = "/dvmdb/adom/{adom}/workspace/unlock/".format(adom=adom)
+        else:
+            url = "/dvmdb/adom/root/workspace/unlock"
+        code, respobj = self.send_request(FAZMethods.EXEC, self._tools.format_request(FAZMethods.EXEC, url))
+        if code == 0 and respobj["status"]["message"].lower() == "ok":
+            self.remove_adom_from_lock_list(adom)
+        return code, respobj
+
+    def commit_changes(self, adom=None, aux=False, *args, **kwargs):
+        """
+        Commits changes to an ADOM
+        """
+        if adom:
+            if aux:
+                url = "/pm/config/adom/{adom}/workspace/commit".format(adom=adom)
+            else:
+                if adom.lower() == "global":
+                    url = "/dvmdb/global/workspace/commit/"
+                else:
+                    url = "/dvmdb/adom/{adom}/workspace/commit".format(adom=adom)
+        else:
+            url = "/dvmdb/adom/root/workspace/commit"
+        return self.send_request(FAZMethods.EXEC, self._tools.format_request(FAZMethods.EXEC, url))
+
+    def get_lock_info(self, adom=None):
+        """
+        Gets ADOM lock info so it can be displayed with the error messages. Or if determined to be locked by ansible
+        for some reason, then unlock it.
+        """
+        if not adom or adom == "root":
+            url = "/dvmdb/adom/root/workspace/lockinfo"
+        else:
+            if adom.lower() == "global":
+                url = "/dvmdb/global/workspace/lockinfo/"
+            else:
+                url = "/dvmdb/adom/{adom}/workspace/lockinfo/".format(adom=adom)
+        datagram = {}
+        data = self._tools.format_request(FAZMethods.GET, url, **datagram)
+        resp_obj = self.send_request(FAZMethods.GET, data)
+        code = resp_obj[0]
+        if code != 0:
+            self._module.fail_json(msg=("An error occurred trying to get the ADOM Lock Info. Error: " + to_text(resp_obj)))
+        elif code == 0:
+            try:
+                if resp_obj[1]["status"]["message"] == "OK":
+                    self._lock_info = None
+            except Exception:
+                self._lock_info = resp_obj[1]
+        return resp_obj
+
+    def get_adom_list(self):
+        """
+        Gets the list of ADOMs for the FortiAnalyzer
+        """
+        if self.uses_adoms:
+            url = "/dvmdb/adom"
+            datagram = {}
+            data = self._tools.format_request(FAZMethods.GET, url, **datagram)
+            resp_obj = self.send_request(FAZMethods.GET, data)
+            code = resp_obj[0]
+            if code != 0:
+                self._module.fail_json(msg=("An error occurred trying to get the ADOM Info. Error: " + to_text(resp_obj)))
+            elif code == 0:
+                num_of_adoms = len(resp_obj[1])
+                append_list = ['root', ]
+                for adom in resp_obj[1]:
+                    if adom["tab_status"] != "":
+                        append_list.append(to_text(adom["name"]))
+                self._adom_list = append_list
+            return resp_obj
+
+    def get_locked_adom_list(self):
+        """
+        Gets the list of locked adoms
+        """
+        try:
+            locked_list = list()
+            locked_by_user_list = list()
+            for adom in self._adom_list:
+                adom_lock_info = self.get_lock_info(adom=adom)
+                try:
+                    if adom_lock_info[1]["status"]["message"] == "OK":
+                        continue
+                except Exception:
+                    pass
+                try:
+                    if adom_lock_info[1][0]["lock_user"]:
+                        locked_list.append(to_text(adom))
+                    if adom_lock_info[1][0]["lock_user"] == self._logged_in_user:
+                        locked_by_user_list.append({"adom": to_text(adom), "user": to_text(adom_lock_info[1][0]["lock_user"])})
+                except Exception as err:
+                    raise FAZBaseException(err)
+            self._locked_adom_list = locked_list
+            self._locked_adoms_by_user = locked_by_user_list
+
+        except Exception as err:
+            raise FAZBaseException(msg=("An error occurred while trying to get the locked adom list. Error: "
+                                        + to_text(err)))
+
+    #################################
+    # END DATABASE LOCK CONTEXT CODE
+    #################################

plugins/httpapi/fortimanager.py

@@ -0,0 +1,451 @@
+# Copyright (c) 2018 Fortinet and/or its affiliates.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import (absolute_import, division, print_function)
+
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+author:
+    - Luke Weighall (@lweighall)
+    - Andrew Welsh (@Ghilli3)
+    - Jim Huber (@p4r4n0y1ng)
+httpapi : fortimanager
+short_description: HttpApi Plugin for Fortinet FortiManager Appliance or VM.
+description:
+  - This HttpApi plugin provides methods to connect to Fortinet FortiManager Appliance or VM via JSON RPC API.
+'''
+
+import json
+from ansible.plugins.httpapi import HttpApiBase
+from ansible.module_utils.basic import to_text
+from ansible_collections.fortinet.fortios.plugins.module_utils.network.fortimanager.common import BASE_HEADERS
+from ansible_collections.fortinet.fortios.plugins.module_utils.network.fortimanager.common import FMGBaseException
+from ansible_collections.fortinet.fortios.plugins.module_utils.network.fortimanager.common import FMGRCommon
+from ansible_collections.fortinet.fortios.plugins.module_utils.network.fortimanager.common import FMGRMethods
+
+
+class HttpApi(HttpApiBase):
+    def __init__(self, connection):
+        super(HttpApi, self).__init__(connection)
+        self._req_id = 0
+        self._sid = None
+        self._url = "/jsonrpc"
+        self._host = None
+        self._tools = FMGRCommon
+        self._debug = False
+        self._connected_fmgr = None
+        self._last_response_msg = None
+        self._last_response_code = None
+        self._last_data_payload = None
+        self._last_url = None
+        self._last_response_raw = None
+        self._locked_adom_list = list()
+        self._locked_adoms_by_user = list()
+        self._uses_workspace = False
+        self._uses_adoms = False
+        self._adom_list = list()
+        self._logged_in_user = None
+
+    def set_become(self, become_context):
+        """
+        ELEVATION IS NOT REQUIRED ON FORTINET DEVICES - SKIPPED.
+        :param become_context: Unused input.
+        :return: None
+        """
+        return None
+
+    def update_auth(self, response, response_data):
+        """
+        TOKENS ARE NOT USED SO NO NEED TO UPDATE AUTH.
+        :param response: Unused input.
+        :param response_data Unused_input.
+        :return: None
+        """
+        return None
+
+    def login(self, username, password):
+
+        """
+        This function will log the plugin into FortiManager, and return the results.
+        :param username: Username of FortiManager Admin
+        :param password: Password of FortiManager Admin
+
+        :return: Dictionary of status if it logged in or not.
+        """
+        self._logged_in_user = username
+        self.send_request(FMGRMethods.EXEC, self._tools.format_request(FMGRMethods.EXEC, "sys/login/user",
+                                                                       passwd=password, user=username, ))
+
+        if "FortiManager object connected to FortiManager" in self.__str__():
+            # If Login worked, then inspect the FortiManager for Workspace Mode, and it's system information.
+            self.inspect_fmgr()
+            return
+        else:
+            raise FMGBaseException(msg="Unknown error while logging in...connection was lost during login operation...."
+                                       " Exiting")
+
+    def inspect_fmgr(self):
+        # CHECK FOR WORKSPACE MODE TO SEE IF WE HAVE TO ENABLE ADOM LOCKS
+        status = self.get_system_status()
+        if status[0] == -11:
+            # THE CONNECTION GOT LOST SOMEHOW, REMOVE THE SID AND REPORT BAD LOGIN
+            self.logout()
+            raise FMGBaseException(msg="Error -11 -- the Session ID was likely malformed somehow. Contact authors."
+                                       " Exiting")
+        elif status[0] == 0:
+            try:
+                self.check_mode()
+                if self._uses_adoms:
+                    self.get_adom_list()
+                if self._uses_workspace:
+                    self.get_locked_adom_list()
+                self._connected_fmgr = status[1]
+                self._host = self._connected_fmgr["Hostname"]
+            except BaseException:
+                pass
+        return
+
+    def logout(self):
+        """
+        This function will logout of the FortiManager.
+        """
+        if self.sid is not None:
+            # IF WE WERE USING WORKSPACES, THEN CLEAN UP OUR LOCKS IF THEY STILL EXIST
+            if self.uses_workspace:
+                self.get_lock_info()
+                self.run_unlock()
+            ret_code, response = self.send_request(FMGRMethods.EXEC,
+                                                   self._tools.format_request(FMGRMethods.EXEC, "sys/logout"))
+            self.sid = None
+            return ret_code, response
+
+    def send_request(self, method, params):
+        """
+        Responsible for actual sending of data to the connection httpapi base plugin. Does some formatting too.
+        :param params: A formatted dictionary that was returned by self.common_datagram_params()
+        before being called here.
+        :param method: The preferred API Request method (GET, ADD, POST, etc....)
+        :type method: basestring
+
+        :return: Dictionary of status, if it logged in or not.
+        """
+        try:
+            if self.sid is None and params[0]["url"] != "sys/login/user":
+                try:
+                    self.connection._connect()
+                except Exception as err:
+                    raise FMGBaseException(
+                        msg="An problem happened with the httpapi plugin self-init connection process. "
+                            "Error: " + to_text(err))
+        except IndexError:
+            raise FMGBaseException("An attempt was made at communicating with a FMG with "
+                                   "no valid session and an incorrectly formatted request.")
+        except Exception as err:
+            raise FMGBaseException("An attempt was made at communicating with a FMG with "
+                                   "no valid session and an unexpected error was discovered. \n Error: " + to_text(err))
+
+        self._update_request_id()
+        json_request = {
+            "method": method,
+            "params": params,
+            "session": self.sid,
+            "id": self.req_id,
+            "verbose": 1
+        }
+        data = json.dumps(json_request, ensure_ascii=False).replace('\\\\', '\\')
+        try:
+            # Sending URL and Data in Unicode, per Ansible Specifications for Connection Plugins
+            response, response_data = self.connection.send(path=to_text(self._url), data=to_text(data),
+                                                           headers=BASE_HEADERS)
+            # Get Unicode Response - Must convert from StringIO to unicode first so we can do a replace function below
+            result = json.loads(to_text(response_data.getvalue()))
+            self._update_self_from_response(result, self._url, data)
+            return self._handle_response(result)
+        except Exception as err:
+            raise FMGBaseException(err)
+
+    def _handle_response(self, response):
+        self._set_sid(response)
+        if isinstance(response["result"], list):
+            result = response["result"][0]
+        else:
+            result = response["result"]
+        if "data" in result:
+            return result["status"]["code"], result["data"]
+        else:
+            return result["status"]["code"], result
+
+    def _update_self_from_response(self, response, url, data):
+        self._last_response_raw = response
+        if isinstance(response["result"], list):
+            result = response["result"][0]
+        else:
+            result = response["result"]
+        if "status" in result:
+            self._last_response_code = result["status"]["code"]
+            self._last_response_msg = result["status"]["message"]
+            self._last_url = url
+            self._last_data_payload = data
+
+    def _set_sid(self, response):
+        if self.sid is None and "session" in response:
+            self.sid = response["session"]
+
+    def return_connected_fmgr(self):
+        """
+        Returns the data stored under self._connected_fmgr
+
+        :return: dict
+        """
+        try:
+            if self._connected_fmgr:
+                return self._connected_fmgr
+        except Exception:
+            raise FMGBaseException("Couldn't Retrieve Connected FMGR Stats")
+
+    def get_system_status(self):
+        """
+        Returns the system status page from the FortiManager, for logging and other uses.
+        return: status
+        """
+        status = self.send_request(FMGRMethods.GET, self._tools.format_request(FMGRMethods.GET, "sys/status"))
+        return status
+
+    @property
+    def debug(self):
+        return self._debug
+
+    @debug.setter
+    def debug(self, val):
+        self._debug = val
+
+    @property
+    def req_id(self):
+        return self._req_id
+
+    @req_id.setter
+    def req_id(self, val):
+        self._req_id = val
+
+    def _update_request_id(self, reqid=0):
+        self.req_id = reqid if reqid != 0 else self.req_id + 1
+
+    @property
+    def sid(self):
+        return self._sid
+
+    @sid.setter
+    def sid(self, val):
+        self._sid = val
+
+    def __str__(self):
+        if self.sid is not None and self.connection._url is not None:
+            return "FortiManager object connected to FortiManager: " + to_text(self.connection._url)
+        return "FortiManager object with no valid connection to a FortiManager appliance."
+
+    ##################################
+    # BEGIN DATABASE LOCK CONTEXT CODE
+    ##################################
+
+    @property
+    def uses_workspace(self):
+        return self._uses_workspace
+
+    @uses_workspace.setter
+    def uses_workspace(self, val):
+        self._uses_workspace = val
+
+    @property
+    def uses_adoms(self):
+        return self._uses_adoms
+
+    @uses_adoms.setter
+    def uses_adoms(self, val):
+        self._uses_adoms = val
+
+    def add_adom_to_lock_list(self, adom):
+        if adom not in self._locked_adom_list:
+            self._locked_adom_list.append(adom)
+
+    def remove_adom_from_lock_list(self, adom):
+        if adom in self._locked_adom_list:
+            self._locked_adom_list.remove(adom)
+
+    def check_mode(self):
+        """
+        Checks FortiManager for the use of Workspace mode
+        """
+        url = "/cli/global/system/global"
+        code, resp_obj = self.send_request(FMGRMethods.GET,
+                                           self._tools.format_request(FMGRMethods.GET,
+                                                                      url,
+                                                                      fields=["workspace-mode", "adom-status"]))
+        try:
+            if resp_obj["workspace-mode"] == "workflow":
+                self.uses_workspace = True
+            elif resp_obj["workspace-mode"] == "disabled":
+                self.uses_workspace = False
+        except KeyError:
+            raise FMGBaseException(msg="Couldn't determine workspace-mode in the plugin")
+        try:
+            if resp_obj["adom-status"] in [1, "enable"]:
+                self.uses_adoms = True
+            else:
+                self.uses_adoms = False
+        except KeyError:
+            raise FMGBaseException(msg="Couldn't determine adom-status in the plugin")
+
+    def run_unlock(self):
+        """
+        Checks for ADOM status, if locked, it will unlock
+        """
+        for adom_locked in self._locked_adoms_by_user:
+            adom = adom_locked["adom"]
+            self.unlock_adom(adom)
+
+    def lock_adom(self, adom=None, *args, **kwargs):
+        """
+        Locks an ADOM for changes
+        """
+        if adom:
+            if adom.lower() == "global":
+                url = "/dvmdb/global/workspace/lock/"
+            else:
+                url = "/dvmdb/adom/{adom}/workspace/lock/".format(adom=adom)
+        else:
+            url = "/dvmdb/adom/root/workspace/lock"
+        code, respobj = self.send_request(FMGRMethods.EXEC, self._tools.format_request(FMGRMethods.EXEC, url))
+        if code == 0 and respobj["status"]["message"].lower() == "ok":
+            self.add_adom_to_lock_list(adom)
+        return code, respobj
+
+    def unlock_adom(self, adom=None, *args, **kwargs):
+        """
+        Unlocks an ADOM after changes
+        """
+        if adom:
+            if adom.lower() == "global":
+                url = "/dvmdb/global/workspace/unlock/"
+            else:
+                url = "/dvmdb/adom/{adom}/workspace/unlock/".format(adom=adom)
+        else:
+            url = "/dvmdb/adom/root/workspace/unlock"
+        code, respobj = self.send_request(FMGRMethods.EXEC, self._tools.format_request(FMGRMethods.EXEC, url))
+        if code == 0 and respobj["status"]["message"].lower() == "ok":
+            self.remove_adom_from_lock_list(adom)
+        return code, respobj
+
+    def commit_changes(self, adom=None, aux=False, *args, **kwargs):
+        """
+        Commits changes to an ADOM
+        """
+        if adom:
+            if aux:
+                url = "/pm/config/adom/{adom}/workspace/commit".format(adom=adom)
+            else:
+                if adom.lower() == "global":
+                    url = "/dvmdb/global/workspace/commit/"
+                else:
+                    url = "/dvmdb/adom/{adom}/workspace/commit".format(adom=adom)
+        else:
+            url = "/dvmdb/adom/root/workspace/commit"
+        return self.send_request(FMGRMethods.EXEC, self._tools.format_request(FMGRMethods.EXEC, url))
+
+    def get_lock_info(self, adom=None):
+        """
+        Gets ADOM lock info so it can be displayed with the error messages. Or if determined to be locked by ansible
+        for some reason, then unlock it.
+        """
+        if not adom or adom == "root":
+            url = "/dvmdb/adom/root/workspace/lockinfo"
+        else:
+            if adom.lower() == "global":
+                url = "/dvmdb/global/workspace/lockinfo/"
+            else:
+                url = "/dvmdb/adom/{adom}/workspace/lockinfo/".format(adom=adom)
+        datagram = {}
+        data = self._tools.format_request(FMGRMethods.GET, url, **datagram)
+        resp_obj = self.send_request(FMGRMethods.GET, data)
+        code = resp_obj[0]
+        if code != 0:
+            self._module.fail_json(msg=("An error occurred trying to get the ADOM Lock Info. "
+                                        "Error: " + to_text(resp_obj)))
+        elif code == 0:
+            try:
+                if resp_obj[1]["status"]["message"] == "OK":
+                    self._lock_info = None
+            except Exception:
+                self._lock_info = resp_obj[1]
+        return resp_obj
+
+    def get_adom_list(self):
+        """
+        Gets the list of ADOMs for the FortiManager
+        """
+        if self.uses_adoms:
+            url = "/dvmdb/adom"
+            datagram = {}
+            data = self._tools.format_request(FMGRMethods.GET, url, **datagram)
+            resp_obj = self.send_request(FMGRMethods.GET, data)
+            code = resp_obj[0]
+            if code != 0:
+                self._module.fail_json(msg=("An error occurred trying to get the ADOM Info. "
+                                            "Error: " + to_text(resp_obj)))
+            elif code == 0:
+                num_of_adoms = len(resp_obj[1])
+                append_list = ['root', ]
+                for adom in resp_obj[1]:
+                    if adom["tab_status"] != "":
+                        append_list.append(to_text(adom["name"]))
+                self._adom_list = append_list
+            return resp_obj
+
+    def get_locked_adom_list(self):
+        """
+        Gets the list of locked adoms
+        """
+        try:
+            locked_list = list()
+            locked_by_user_list = list()
+            for adom in self._adom_list:
+                adom_lock_info = self.get_lock_info(adom=adom)
+                try:
+                    if adom_lock_info[1]["status"]["message"] == "OK":
+                        continue
+                except IndexError as err:
+                    pass
+                try:
+                    if adom_lock_info[1][0]["lock_user"]:
+                        locked_list.append(to_text(adom))
+                    if adom_lock_info[1][0]["lock_user"] == self._logged_in_user:
+                        locked_by_user_list.append({"adom": to_text(adom),
+                                                    "user": to_text(adom_lock_info[1][0]["lock_user"])})
+                except Exception as err:
+                    raise FMGBaseException(err)
+            self._locked_adom_list = locked_list
+            self._locked_adoms_by_user = locked_by_user_list
+
+        except Exception as err:
+            raise FMGBaseException(msg=("An error occurred while trying to get the locked adom list. Error: "
+                                        + to_text(err)))
+
+    ################################
+    # END DATABASE LOCK CONTEXT CODE
+    ################################

plugins/httpapi/ftd.py

@@ -0,0 +1,386 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import (absolute_import, division, print_function)
+
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+author: Ansible Networking Team
+httpapi : ftd
+short_description: HttpApi Plugin for Cisco ASA Firepower device
+description:
+  - This HttpApi plugin provides methods to connect to Cisco ASA firepower
+    devices over a HTTP(S)-based api.
+options:
+  token_path:
+    type: str
+    description:
+      - Specifies the api token path of the FTD device
+    vars:
+      - name: ansible_httpapi_ftd_token_path
+  spec_path:
+    type: str
+    description:
+      - Specifies the api spec path of the FTD device
+    default: '/apispec/ngfw.json'
+    vars:
+      - name: ansible_httpapi_ftd_spec_path
+'''
+
+import json
+import os
+import re
+
+from ansible import __version__ as ansible_version
+
+from ansible.module_utils.basic import to_text
+from ansible.errors import AnsibleConnectionFailure
+from ansible_collections.community.general.plugins.module_utils.network.ftd.fdm_swagger_client import FdmSwaggerParser, SpecProp, FdmSwaggerValidator
+from ansible_collections.community.general.plugins.module_utils.network.ftd.common import HTTPMethod, ResponseParams
+from ansible.module_utils.six.moves.urllib.error import HTTPError
+from ansible.module_utils.six.moves.urllib.parse import urlencode
+from ansible.plugins.httpapi import HttpApiBase
+from urllib3 import encode_multipart_formdata
+from urllib3.fields import RequestField
+from ansible.module_utils.connection import ConnectionError
+
+BASE_HEADERS = {
+    'Content-Type': 'application/json',
+    'Accept': 'application/json',
+    'User-Agent': 'FTD Ansible/%s' % ansible_version
+}
+
+TOKEN_EXPIRATION_STATUS_CODE = 408
+UNAUTHORIZED_STATUS_CODE = 401
+API_TOKEN_PATH_OPTION_NAME = 'token_path'
+TOKEN_PATH_TEMPLATE = '/api/fdm/{0}/fdm/token'
+GET_API_VERSIONS_PATH = '/api/versions'
+DEFAULT_API_VERSIONS = ['v2', 'v1']
+
+INVALID_API_TOKEN_PATH_MSG = ('The API token path is incorrect. Please, check correctness of '
+                              'the `ansible_httpapi_ftd_token_path` variable in the inventory file.')
+MISSING_API_TOKEN_PATH_MSG = ('Ansible could not determine the API token path automatically. Please, '
+                              'specify the `ansible_httpapi_ftd_token_path` variable in the inventory file.')
+
+
+class HttpApi(HttpApiBase):
+    def __init__(self, connection):
+        super(HttpApi, self).__init__(connection)
+        self.connection = connection
+        self.access_token = None
+        self.refresh_token = None
+        self._api_spec = None
+        self._api_validator = None
+        self._ignore_http_errors = False
+
+    def login(self, username, password):
+        def request_token_payload(username, password):
+            return {
+                'grant_type': 'password',
+                'username': username,
+                'password': password
+            }
+
+        def refresh_token_payload(refresh_token):
+            return {
+                'grant_type': 'refresh_token',
+                'refresh_token': refresh_token
+            }
+
+        if self.refresh_token:
+            payload = refresh_token_payload(self.refresh_token)
+        elif username and password:
+            payload = request_token_payload(username, password)
+        else:
+            raise AnsibleConnectionFailure('Username and password are required for login in absence of refresh token')
+
+        response = self._lookup_login_url(payload)
+
+        try:
+            self.refresh_token = response['refresh_token']
+            self.access_token = response['access_token']
+            self.connection._auth = {'Authorization': 'Bearer %s' % self.access_token}
+        except KeyError:
+            raise ConnectionError(
+                'Server returned response without token info during connection authentication: %s' % response)
+
+    def _lookup_login_url(self, payload):
+        """ Try to find correct login URL and get api token using this URL.
+
+        :param payload: Token request payload
+        :type payload: dict
+        :return: token generation response
+        """
+        preconfigured_token_path = self._get_api_token_path()
+        if preconfigured_token_path:
+            token_paths = [preconfigured_token_path]
+        else:
+            token_paths = self._get_known_token_paths()
+
+        for url in token_paths:
+            try:
+                response = self._send_login_request(payload, url)
+
+            except ConnectionError as e:
+                self.connection.queue_message('vvvv', 'REST:request to %s failed because of connection error: %s ' % (
+                    url, e))
+                # In the case of ConnectionError caused by HTTPError we should check response code.
+                # Response code 400 returned in case of invalid credentials so we should stop attempts to log in and
+                # inform the user.
+                if hasattr(e, 'http_code') and e.http_code == 400:
+                    raise
+            else:
+                if not preconfigured_token_path:
+                    self._set_api_token_path(url)
+                return response
+
+        raise ConnectionError(INVALID_API_TOKEN_PATH_MSG if preconfigured_token_path else MISSING_API_TOKEN_PATH_MSG)
+
+    def _send_login_request(self, payload, url):
+        self._display(HTTPMethod.POST, 'login', url)
+        response, response_data = self._send_auth_request(
+            url, json.dumps(payload), method=HTTPMethod.POST, headers=BASE_HEADERS
+        )
+        self._display(HTTPMethod.POST, 'login:status_code', response.getcode())
+
+        response = self._response_to_json(self._get_response_value(response_data))
+        return response
+
+    def logout(self):
+        auth_payload = {
+            'grant_type': 'revoke_token',
+            'access_token': self.access_token,
+            'token_to_revoke': self.refresh_token
+        }
+
+        url = self._get_api_token_path()
+
+        self._display(HTTPMethod.POST, 'logout', url)
+        response, dummy = self._send_auth_request(url, json.dumps(auth_payload), method=HTTPMethod.POST,
+                                                  headers=BASE_HEADERS)
+        self._display(HTTPMethod.POST, 'logout:status_code', response.getcode())
+
+        self.refresh_token = None
+        self.access_token = None
+
+    def _send_auth_request(self, path, data, **kwargs):
+        error_msg_prefix = 'Server returned an error during authentication request'
+        return self._send_service_request(path, error_msg_prefix, data=data, **kwargs)
+
+    def _send_service_request(self, path, error_msg_prefix, data=None, **kwargs):
+        try:
+            self._ignore_http_errors = True
+            return self.connection.send(path, data, **kwargs)
+        except HTTPError as e:
+            # HttpApi connection does not read the error response from HTTPError, so we do it here and wrap it up in
+            # ConnectionError, so the actual error message is displayed to the user.
+            error_msg = self._response_to_json(to_text(e.read()))
+            raise ConnectionError('%s: %s' % (error_msg_prefix, error_msg), http_code=e.code)
+        finally:
+            self._ignore_http_errors = False
+
+    def update_auth(self, response, response_data):
+        # With tokens, authentication should not be checked and updated on each request
+        return None
+
+    def send_request(self, url_path, http_method, body_params=None, path_params=None, query_params=None):
+        url = construct_url_path(url_path, path_params, query_params)
+        data = json.dumps(body_params) if body_params else None
+        try:
+            self._display(http_method, 'url', url)
+            if data:
+                self._display(http_method, 'data', data)
+
+            response, response_data = self.connection.send(url, data, method=http_method, headers=BASE_HEADERS)
+
+            value = self._get_response_value(response_data)
+            self._display(http_method, 'response', value)
+
+            return {
+                ResponseParams.SUCCESS: True,
+                ResponseParams.STATUS_CODE: response.getcode(),
+                ResponseParams.RESPONSE: self._response_to_json(value)
+            }
+        # Being invoked via JSON-RPC, this method does not serialize and pass HTTPError correctly to the method caller.
+        # Thus, in order to handle non-200 responses, we need to wrap them into a simple structure and pass explicitly.
+        except HTTPError as e:
+            error_msg = to_text(e.read())
+            self._display(http_method, 'error', error_msg)
+            return {
+                ResponseParams.SUCCESS: False,
+                ResponseParams.STATUS_CODE: e.code,
+                ResponseParams.RESPONSE: self._response_to_json(error_msg)
+            }
+
+    def upload_file(self, from_path, to_url):
+        url = construct_url_path(to_url)
+        self._display(HTTPMethod.POST, 'upload', url)
+        with open(from_path, 'rb') as src_file:
+            rf = RequestField('fileToUpload', src_file.read(), os.path.basename(src_file.name))
+            rf.make_multipart()
+            body, content_type = encode_multipart_formdata([rf])
+
+            headers = dict(BASE_HEADERS)
+            headers['Content-Type'] = content_type
+            headers['Content-Length'] = len(body)
+
+            dummy, response_data = self.connection.send(url, data=body, method=HTTPMethod.POST, headers=headers)
+            value = self._get_response_value(response_data)
+            self._display(HTTPMethod.POST, 'upload:response', value)
+            return self._response_to_json(value)
+
+    def download_file(self, from_url, to_path, path_params=None):
+        url = construct_url_path(from_url, path_params=path_params)
+        self._display(HTTPMethod.GET, 'download', url)
+        response, response_data = self.connection.send(url, data=None, method=HTTPMethod.GET, headers=BASE_HEADERS)
+
+        if os.path.isdir(to_path):
+            filename = extract_filename_from_headers(response.info())
+            to_path = os.path.join(to_path, filename)
+
+        with open(to_path, "wb") as output_file:
+            output_file.write(response_data.getvalue())
+        self._display(HTTPMethod.GET, 'downloaded', to_path)
+
+    def handle_httperror(self, exc):
+        is_auth_related_code = exc.code == TOKEN_EXPIRATION_STATUS_CODE or exc.code == UNAUTHORIZED_STATUS_CODE
+        if not self._ignore_http_errors and is_auth_related_code:
+            self.connection._auth = None
+            self.login(self.connection.get_option('remote_user'), self.connection.get_option('password'))
+            return True
+        # False means that the exception will be passed further to the caller
+        return False
+
+    def _display(self, http_method, title, msg=''):
+        self.connection.queue_message('vvvv', 'REST:%s:%s:%s\n%s' % (http_method, self.connection._url, title, msg))
+
+    @staticmethod
+    def _get_response_value(response_data):
+        return to_text(response_data.getvalue())
+
+    def _get_api_spec_path(self):
+        return self.get_option('spec_path')
+
+    def _get_known_token_paths(self):
+        """Generate list of token generation urls based on list of versions supported by device(if exposed via API) or
+        default list of API versions.
+
+        :returns: list of token generation urls
+        :rtype: generator
+        """
+        try:
+            api_versions = self._get_supported_api_versions()
+        except ConnectionError:
+            # API versions API is not supported we need to check all known version
+            api_versions = DEFAULT_API_VERSIONS
+
+        return [TOKEN_PATH_TEMPLATE.format(version) for version in api_versions]
+
+    def _get_supported_api_versions(self):
+        """
+        Fetch list of API versions supported by device.
+
+        :return: list of API versions suitable for device
+        :rtype: list
+        """
+        # Try to fetch supported API version
+        http_method = HTTPMethod.GET
+        response, response_data = self._send_service_request(
+            path=GET_API_VERSIONS_PATH,
+            error_msg_prefix="Can't fetch list of supported api versions",
+            method=http_method,
+            headers=BASE_HEADERS
+        )
+
+        value = self._get_response_value(response_data)
+        self._display(http_method, 'response', value)
+        api_versions_info = self._response_to_json(value)
+        return api_versions_info["supportedVersions"]
+
+    def _get_api_token_path(self):
+        return self.get_option(API_TOKEN_PATH_OPTION_NAME)
+
+    def _set_api_token_path(self, url):
+        return self.set_option(API_TOKEN_PATH_OPTION_NAME, url)
+
+    @staticmethod
+    def _response_to_json(response_text):
+        try:
+            return json.loads(response_text) if response_text else {}
+        # JSONDecodeError only available on Python 3.5+
+        except getattr(json.decoder, 'JSONDecodeError', ValueError):
+            raise ConnectionError('Invalid JSON response: %s' % response_text)
+
+    def get_operation_spec(self, operation_name):
+        return self.api_spec[SpecProp.OPERATIONS].get(operation_name, None)
+
+    def get_operation_specs_by_model_name(self, model_name):
+        if model_name:
+            return self.api_spec[SpecProp.MODEL_OPERATIONS].get(model_name, None)
+        else:
+            return None
+
+    def get_model_spec(self, model_name):
+        return self.api_spec[SpecProp.MODELS].get(model_name, None)
+
+    def validate_data(self, operation_name, data):
+        return self.api_validator.validate_data(operation_name, data)
+
+    def validate_query_params(self, operation_name, params):
+        return self.api_validator.validate_query_params(operation_name, params)
+
+    def validate_path_params(self, operation_name, params):
+        return self.api_validator.validate_path_params(operation_name, params)
+
+    @property
+    def api_spec(self):
+        if self._api_spec is None:
+            spec_path_url = self._get_api_spec_path()
+            response = self.send_request(url_path=spec_path_url, http_method=HTTPMethod.GET)
+            if response[ResponseParams.SUCCESS]:
+                self._api_spec = FdmSwaggerParser().parse_spec(response[ResponseParams.RESPONSE])
+            else:
+                raise ConnectionError('Failed to download API specification. Status code: %s. Response: %s' % (
+                    response[ResponseParams.STATUS_CODE], response[ResponseParams.RESPONSE]))
+        return self._api_spec
+
+    @property
+    def api_validator(self):
+        if self._api_validator is None:
+            self._api_validator = FdmSwaggerValidator(self.api_spec)
+        return self._api_validator
+
+
+def construct_url_path(path, path_params=None, query_params=None):
+    url = path
+    if path_params:
+        url = url.format(**path_params)
+    if query_params:
+        url += "?" + urlencode(query_params)
+    return url
+
+
+def extract_filename_from_headers(response_info):
+    content_header_regex = r'attachment; ?filename="?([^"]+)'
+    match = re.match(content_header_regex, response_info.get('Content-Disposition'))
+    if match:
+        return match.group(1)
+    else:
+        raise ValueError("No appropriate Content-Disposition header is specified.")