diff --git a/lib/ansible/module_utils/netapp.py b/lib/ansible/module_utils/netapp.py index 92b8ddb8b4..ddf2bb8a43 100644 --- a/lib/ansible/module_utils/netapp.py +++ b/lib/ansible/module_utils/netapp.py @@ -30,9 +30,13 @@ import json import os -from ansible.module_utils.six.moves.urllib.error import HTTPError +from pprint import pformat +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.six.moves.urllib.error import HTTPError, URLError from ansible.module_utils.urls import open_url from ansible.module_utils.api import basic_auth_argument_spec +from ansible.module_utils._text import to_native + try: from ansible.module_utils.ansible_release import __version__ as ansible_version except ImportError: @@ -45,6 +49,11 @@ except ImportError: HAS_NETAPP_LIB = False import ssl +try: + from urlparse import urlparse, urlunparse +except ImportError: + from urllib.parse import urlparse, urlunparse + HAS_SF_SDK = False SF_BYTE_MAP = dict( @@ -195,33 +204,249 @@ def setup_ontap_zapi(module, vserver=None): def eseries_host_argument_spec(): - """Retrieve a base argument specifiation common to all NetApp E-Series modules""" + """Retrieve a base argument specification common to all NetApp E-Series modules""" argument_spec = basic_auth_argument_spec() argument_spec.update(dict( api_username=dict(type='str', required=True), api_password=dict(type='str', required=True, no_log=True), api_url=dict(type='str', required=True), - ssid=dict(type='str', required=True), - validate_certs=dict(type='bool', required=False, default=True), + ssid=dict(type='str', required=False, default='1'), + validate_certs=dict(type='bool', required=False, default=True) )) return argument_spec +class NetAppESeriesModule(object): + """Base class for all NetApp E-Series modules. + + Provides a set of common methods for NetApp E-Series modules, including version checking, mode (proxy, embedded) + verification, http requests, secure http redirection for embedded web services, and logging setup. + + Be sure to add the following lines in the module's documentation section: + extends_documentation_fragment: + - netapp.eseries + + :param dict(dict) ansible_options: dictionary of ansible option definitions + :param str web_services_version: minimally required web services rest api version (default value: "02.00.0000.0000") + :param bool supports_check_mode: whether the module will support the check_mode capabilities (default=False) + :param list(list) mutually_exclusive: list containing list(s) of mutually exclusive options (optional) + :param list(list) required_if: list containing list(s) containing the option, the option value, and then + a list of required options. (optional) + :param list(list) required_one_of: list containing list(s) of options for which at least one is required. (optional) + :param list(list) required_together: list containing list(s) of options that are required together. (optional) + :param bool log_requests: controls whether to log each request (default: True) + """ + DEFAULT_TIMEOUT = 60 + DEFAULT_SECURE_PORT = "8443" + DEFAULT_REST_API_PATH = "devmgr/v2" + DEFAULT_HEADERS = {"Content-Type": "application/json", "Accept": "application/json", + "netapp-client-type": "Ansible-%s" % ansible_version} + HTTP_AGENT = "Ansible / %s" % ansible_version + SIZE_UNIT_MAP = dict(bytes=1, b=1, kb=1024, mb=1024**2, gb=1024**3, tb=1024**4, + pb=1024**5, eb=1024**6, zb=1024**7, yb=1024**8) + + def __init__(self, ansible_options, web_services_version=None, supports_check_mode=False, + mutually_exclusive=None, required_if=None, required_one_of=None, required_together=None, + log_requests=True): + argument_spec = eseries_host_argument_spec() + argument_spec.update(ansible_options) + + self.module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=supports_check_mode, + mutually_exclusive=mutually_exclusive, required_if=required_if, + required_one_of=required_one_of, required_together=required_together) + + args = self.module.params + self.web_services_version = web_services_version if web_services_version else "02.00.0000.0000" + self.url = args["api_url"] + self.ssid = args["ssid"] + self.creds = dict(url_username=args["api_username"], + url_password=args["api_password"], + validate_certs=args["validate_certs"]) + self.log_requests = log_requests + + self.is_embedded_mode = None + self.web_services_validate = None + + self._tweak_url() + + @property + def _about_url(self): + """Generates the about url based on the supplied web services rest api url. + + :raise AnsibleFailJson: raised when supplied web services rest api is an invalid url. + :return: proxy or embedded about url. + """ + about = list(urlparse(self.url)) + about[2] = "devmgr/utils/about" + return urlunparse(about) + + def _force_secure_url(self): + """Modifies supplied web services rest api to use secure https. + + raise: AnsibleFailJson: raised when the url already utilizes the secure protocol + """ + url_parts = list(urlparse(self.url)) + if "https://" in self.url and ":8443" in self.url: + self.module.fail_json(msg="Secure HTTP protocol already used. URL path [%s]" % self.url) + + url_parts[0] = "https" + url_parts[1] = "%s:8443" % url_parts[1].split(":")[0] + + self.url = urlunparse(url_parts) + if not self.url.endswith("/"): + self.url += "/" + + self.module.warn("forced use of the secure protocol: %s" % self.url) + + def _tweak_url(self): + """Adjust the rest api url is necessary. + + :raise AnsibleFailJson: raised when self.url fails to have a hostname or ipv4 address. + """ + # ensure the protocol is either http or https + if self.url.split("://")[0] not in ["https", "http"]: + + self.url = self.url.split("://")[1] if "://" in self.url else self.url + + if ":8080" in self.url: + self.url = "http://%s" % self.url + else: + self.url = "https://%s" % self.url + + # parse url and verify protocol, port and path are consistent with required web services rest api url. + url_parts = list(urlparse(self.url)) + if url_parts[1] == "": + self.module.fail_json(msg="Failed to provide a valid hostname or IP address. URL [%s]." % self.url) + + split_hostname = url_parts[1].split(":") + if url_parts[0] not in ["https", "http"] or (len(split_hostname) == 2 and split_hostname[1] != "8080"): + if len(split_hostname) == 2 and split_hostname[1] == "8080": + url_parts[0] = "http" + url_parts[1] = "%s:8080" % split_hostname[0] + else: + url_parts[0] = "https" + url_parts[1] = "%s:8443" % split_hostname[0] + elif len(split_hostname) == 1: + if url_parts[0] == "https": + url_parts[1] = "%s:8443" % split_hostname[0] + elif url_parts[0] == "http": + url_parts[1] = "%s:8080" % split_hostname[0] + + if url_parts[2] == "" or url_parts[2] != self.DEFAULT_REST_API_PATH: + url_parts[2] = self.DEFAULT_REST_API_PATH + + self.url = urlunparse(url_parts) + if not self.url.endswith("/"): + self.url += "/" + + self.module.log("valid url: %s" % self.url) + + def _is_web_services_valid(self): + """Verify proxy or embedded web services meets minimum version required for module. + + The minimum required web services version is evaluated against version supplied through the web services rest + api. AnsibleFailJson exception will be raised when the minimum is not met or exceeded. + + :raise AnsibleFailJson: raised when the contacted api service does not meet the minimum required version. + """ + if not self.web_services_validate: + self.is_embedded() + try: + rc, data = request(self._about_url, timeout=self.DEFAULT_TIMEOUT, + headers=self.DEFAULT_HEADERS, **self.creds) + major, minor, other, revision = data["version"].split(".") + minimum_major, minimum_minor, other, minimum_revision = self.web_services_version.split(".") + + if not (major > minimum_major or + (major == minimum_major and minor > minimum_minor) or + (major == minimum_major and minor == minimum_minor and revision >= minimum_revision)): + self.module.fail_json( + msg="Web services version does not meet minimum version required. Current version: [%s]." + " Version required: [%s]." % (data["version"], self.web_services_version)) + except Exception as error: + self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]." + " Error [%s]." % (self.ssid, to_native(error))) + + self.module.warn("Web services rest api version met the minimum required version.") + self.web_services_validate = True + + return self.web_services_validate + + def is_embedded(self, retry=True): + """Determine whether web services server is the embedded web services. + + If web services about endpoint fails based on an URLError then the request will be attempted again using + secure http. + + :raise AnsibleFailJson: raised when web services about endpoint failed to be contacted. + :return bool: whether contacted web services is running from storage array (embedded) or from a proxy. + """ + if self.is_embedded_mode is None: + try: + rc, data = request(self._about_url, timeout=self.DEFAULT_TIMEOUT, + headers=self.DEFAULT_HEADERS, **self.creds) + self.is_embedded_mode = not data["runningAsProxy"] + except URLError as error: + if not retry: + self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]." + " Error [%s]." % (self.ssid, to_native(error))) + self.module.warn("Failed to retrieve the webservices about information! Will retry using secure" + " http. Array Id [%s]." % self.ssid) + self._force_secure_url() + return self.is_embedded(retry=False) + except Exception as error: + self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]." + " Error [%s]." % (self.ssid, to_native(error))) + + return self.is_embedded_mode + + def request(self, path, data=None, method='GET', ignore_errors=False): + """Issue an HTTP request to a url, retrieving an optional JSON response. + + :param str path: web services rest api endpoint path (Example: storage-systems/1/graph). Note that when the + full url path is specified then that will be used without supplying the protocol, hostname, port and rest path. + :param data: data required for the request (data may be json or any python structured data) + :param str method: request method such as GET, POST, DELETE. + :param bool ignore_errors: forces the request to ignore any raised exceptions. + """ + if self._is_web_services_valid(): + url = list(urlparse(path.strip("/"))) + if url[2] == "": + self.module.fail_json(msg="Web services rest api endpoint path must be specified. Path [%s]." % path) + + # if either the protocol or hostname/port are missing then add them. + if url[0] == "" or url[1] == "": + url[0], url[1] = list(urlparse(self.url))[:2] + + # add rest api path if the supplied path does not begin with it. + if not all([word in url[2].split("/")[:2] for word in self.DEFAULT_REST_API_PATH.split("/")]): + if not url[2].startswith("/"): + url[2] = "/" + url[2] + url[2] = self.DEFAULT_REST_API_PATH + url[2] + + # ensure data is json formatted + if not isinstance(data, str): + data = json.dumps(data) + + if self.log_requests: + self.module.log(pformat(dict(url=urlunparse(url), data=data, method=method))) + + return request(url=urlunparse(url), data=data, method=method, headers=self.DEFAULT_HEADERS, use_proxy=True, + force=False, last_mod_time=None, timeout=self.DEFAULT_TIMEOUT, http_agent=self.HTTP_AGENT, + force_basic_auth=True, ignore_errors=ignore_errors, **self.creds) + + def request(url, data=None, headers=None, method='GET', use_proxy=True, force=False, last_mod_time=None, timeout=10, validate_certs=True, url_username=None, url_password=None, http_agent=None, force_basic_auth=True, ignore_errors=False): """Issue an HTTP request to a url, retrieving an optional JSON response.""" if headers is None: - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - - } + headers = {"Content-Type": "application/json", "Accept": "application/json"} headers.update({"netapp-client-type": "Ansible-%s" % ansible_version}) if not http_agent: - http_agent = "Ansible / %s" % (ansible_version) + http_agent = "Ansible / %s" % ansible_version try: r = open_url(url=url, data=data, headers=headers, method=method, use_proxy=use_proxy, diff --git a/lib/ansible/plugins/doc_fragments/netapp.py b/lib/ansible/plugins/doc_fragments/netapp.py index 624e698dbb..3d5097f67b 100644 --- a/lib/ansible/plugins/doc_fragments/netapp.py +++ b/lib/ansible/plugins/doc_fragments/netapp.py @@ -148,7 +148,8 @@ options: - Should https certificates be validated? type: bool ssid: - required: true + required: false + default: 1 description: - The ID of the array to manage. This value must be unique for each array. diff --git a/test/units/module_utils/test_netapp.py b/test/units/module_utils/test_netapp.py new file mode 100644 index 0000000000..97eeb0fbfe --- /dev/null +++ b/test/units/module_utils/test_netapp.py @@ -0,0 +1,125 @@ +# (c) 2018, NetApp Inc. +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from ansible.module_utils.six.moves.urllib.error import URLError + +from ansible.module_utils.netapp import NetAppESeriesModule +from units.modules.utils import ModuleTestCase, set_module_args, AnsibleFailJson + +__metaclass__ = type +from units.compat import mock + + +class StubNetAppESeriesModule(NetAppESeriesModule): + def __init__(self): + super(StubNetAppESeriesModule, self).__init__(ansible_options={}) + + +class NetappTest(ModuleTestCase): + REQUIRED_PARAMS = {"api_username": "rw", + "api_password": "password", + "api_url": "http://localhost", + "ssid": "1"} + REQ_FUNC = "ansible.module_utils.netapp.request" + + def _set_args(self, args=None): + module_args = self.REQUIRED_PARAMS.copy() + if args is not None: + module_args.update(args) + set_module_args(module_args) + + def test_about_url_pass(self): + """Verify about_url property returns expected about url.""" + test_set = [("http://localhost/devmgr/v2", "http://localhost:8080/devmgr/utils/about"), + ("http://localhost:8443/devmgr/v2", "https://localhost:8443/devmgr/utils/about"), + ("http://localhost:8443/devmgr/v2/", "https://localhost:8443/devmgr/utils/about"), + ("http://localhost:443/something_else", "https://localhost:8443/devmgr/utils/about"), + ("http://localhost:8443", "https://localhost:8443/devmgr/utils/about"), + ("http://localhost", "http://localhost:8080/devmgr/utils/about")] + + for url in test_set: + self._set_args({"api_url": url[0]}) + base = StubNetAppESeriesModule() + self.assertTrue(base._about_url == url[1]) + + def test_is_embedded_embedded_pass(self): + """Verify is_embedded successfully returns True when an embedded web service's rest api is inquired.""" + self._set_args() + with mock.patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": False})): + base = StubNetAppESeriesModule() + self.assertTrue(base.is_embedded()) + with mock.patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": True})): + base = StubNetAppESeriesModule() + self.assertFalse(base.is_embedded()) + + def test_check_web_services_version_pass(self): + """Verify that an acceptable rest api version passes.""" + minimum_required = "02.10.9000.0010" + test_set = ["03.9.9000.0010", "03.10.9000.0009", "02.11.9000.0009", "02.10.9000.0010"] + + self._set_args() + base = StubNetAppESeriesModule() + base.web_services_version = minimum_required + base.is_embedded = lambda: True + for current_version in test_set: + with mock.patch(self.REQ_FUNC, return_value=(200, {"version": current_version})): + self.assertTrue(base._is_web_services_valid()) + + def test_check_web_services_version_fail(self): + """Verify that an unacceptable rest api version fails.""" + minimum_required = "02.10.9000.0010" + test_set = ["02.10.9000.0009", "02.09.9000.0010", "01.10.9000.0010"] + + self._set_args() + base = StubNetAppESeriesModule() + base.web_services_version = minimum_required + base.is_embedded = lambda: True + for current_version in test_set: + with mock.patch(self.REQ_FUNC, return_value=(200, {"version": current_version})): + with self.assertRaisesRegexp(AnsibleFailJson, r"version does not meet minimum version required."): + base._is_web_services_valid() + + def test_is_embedded_fail(self): + """Verify exception is thrown when a web service's rest api fails to return about information.""" + self._set_args() + with mock.patch(self.REQ_FUNC, return_value=Exception()): + with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to retrieve the webservices about information!"): + base = StubNetAppESeriesModule() + base.is_embedded() + with mock.patch(self.REQ_FUNC, side_effect=[URLError(""), Exception()]): + with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to retrieve the webservices about information!"): + base = StubNetAppESeriesModule() + base.is_embedded() + + def test_tweak_url_pass(self): + """Verify a range of valid netapp eseries rest api urls pass.""" + test_set = [("http://localhost/devmgr/v2", "http://localhost:8080/devmgr/v2/"), + ("localhost", "https://localhost:8443/devmgr/v2/"), + ("localhost:8443/devmgr/v2", "https://localhost:8443/devmgr/v2/"), + ("https://localhost/devmgr/v2", "https://localhost:8443/devmgr/v2/"), + ("http://localhost:8443", "https://localhost:8443/devmgr/v2/"), + ("http://localhost:/devmgr/v2", "https://localhost:8443/devmgr/v2/"), + ("http://localhost:8080", "http://localhost:8080/devmgr/v2/"), + ("http://localhost", "http://localhost:8080/devmgr/v2/"), + ("localhost/devmgr/v2", "https://localhost:8443/devmgr/v2/"), + ("localhost/devmgr", "https://localhost:8443/devmgr/v2/"), + ("localhost/devmgr/v3", "https://localhost:8443/devmgr/v2/"), + ("localhost/something", "https://localhost:8443/devmgr/v2/"), + ("ftp://localhost", "https://localhost:8443/devmgr/v2/"), + ("ftp://localhost:8080", "http://localhost:8080/devmgr/v2/"), + ("ftp://localhost/devmgr/v2/", "https://localhost:8443/devmgr/v2/")] + + for test in test_set: + self._set_args({"api_url": test[0]}) + with mock.patch(self.REQ_FUNC, side_effect=[URLError(""), (200, {"runningAsProxy": False})]): + base = StubNetAppESeriesModule() + base._tweak_url() + self.assertTrue(base.url == test[1]) + + def test_check_url_missing_hostname_fail(self): + """Verify exception is thrown when hostname or ip address is missing.""" + self._set_args({"api_url": "http:///devmgr/v2"}) + with mock.patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": True})): + with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to provide a valid hostname or IP address."): + base = StubNetAppESeriesModule() + base._tweak_url()