diff --git a/lib/ansible/module_utils/gcp.py b/lib/ansible/module_utils/gcp.py index f097704486..91c32d58ba 100644 --- a/lib/ansible/module_utils/gcp.py +++ b/lib/ansible/module_utils/gcp.py @@ -32,41 +32,97 @@ import os import traceback from distutils.version import LooseVersion +# libcloud try: import libcloud HAS_LIBCLOUD_BASE = True except ImportError: HAS_LIBCLOUD_BASE = False -def gcp_connect(module, provider, get_driver, user_agent_product, user_agent_version): - """Return a Google Cloud Platform connection.""" - if not HAS_LIBCLOUD_BASE: - module.fail_json(msg='libcloud must be installed to use this module') +# google-python-api +try: + from httplib2 import Http + from oauth2client.service_account import ServiceAccountCredentials + from googleapiclient.http import set_user_agent + HAS_GOOGLE_API_LIB = True +except ImportError: + HAS_GOOGLE_API_LIB = False +# google-cloud-python +try: + import google.cloud.core as _GOOGLE_CLOUD_CORE_CHECK__ + from httplib2 import Http + HAS_GOOGLE_CLOUD_CORE = True +except ImportError: + HAS_GOOGLE_CLOUD_CORE = False + +# Ansible Display object for warnings +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + +def _get_gcp_ansible_credentials(module): + """Helper to fetch creds from AnsibleModule object.""" service_account_email = module.params.get('service_account_email', None) - credentials_file = module.params.get('credentials_file', None) - pem_file = module.params.get('pem_file', None) + # Note: pem_file is discouraged and will be deprecated + credentials_file = module.params.get('pem_file', None) or module.params.get( + 'credentials_file', None) project_id = module.params.get('project_id', None) + return (service_account_email, credentials_file, project_id) + +def _get_gcp_environ_var(var_name, default_value): + """Wrapper around os.environ.get call.""" + return os.environ.get( + var_name, default_value) + +def _get_gcp_environment_credentials(service_account_email, credentials_file, project_id): + """Helper to look in environment variables for credentials.""" # If any of the values are not given as parameters, check the appropriate # environment variables. if not service_account_email: - service_account_email = os.environ.get('GCE_EMAIL', None) - if not project_id: - project_id = os.environ.get('GCE_PROJECT', None) - if not pem_file: - pem_file = os.environ.get('GCE_PEM_FILE_PATH', None) + service_account_email = _get_gcp_environ_var('GCE_EMAIL', None) if not credentials_file: - credentials_file = os.environ.get('GCE_CREDENTIALS_FILE_PATH', pem_file) + credentials_file = _get_gcp_environ_var( + 'GCE_CREDENTIALS_FILE_PATH', None) or _get_gcp_environ_var( + 'GOOGLE_APPLICATION_CREDENTIALS', None) or _get_gcp_environ_var( + 'GCE_PEM_FILE_PATH', None) + if not project_id: + project_id = _get_gcp_environ_var('GCE_PROJECT', None) or _get_gcp_environ_var( + 'GOOGLE_CLOUD_PROJECT', None) + return (service_account_email, credentials_file, project_id) - # If we still don't have one or more of our credentials, attempt to - # get the remaining values from the libcloud secrets file. - if service_account_email is None or pem_file is None: +def _get_gcp_libcloud_credentials(service_account_email=None, credentials_file=None, project_id=None): + """ + Helper to look for libcloud secrets.py file. + + Note: This has an 'additive' effect right now, filling in + vars not specified elsewhere, in order to keep legacy functionality. + This method of specifying credentials will be deprecated, otherwise + we'd look to make it more restrictive with an all-vars-or-nothing approach. + + :param service_account: GCP service account email used to make requests + :type service_account: ``str`` or None + + :param credentials_file: Path on disk to credentials file + :type credentials_file: ``str`` or None + + :param project_id: GCP project ID. + :type project_id: ``str`` or None + + :return: tuple of (service_account, credentials_file, project_id) + :rtype: ``tuple`` of ``str`` + """ + if service_account_email is None or credentials_file is None: try: import secrets + display.deprecated(msg=("secrets file found at '%s'. This method of specifying " + "credentials is deprecated. Please use env vars or " + "Ansible YAML files instead" % (secrets.__file__)), version=2.5) except ImportError: secrets = None - if hasattr(secrets, 'GCE_PARAMS'): if not service_account_email: service_account_email = secrets.GCE_PARAMS[0] @@ -75,34 +131,139 @@ def gcp_connect(module, provider, get_driver, user_agent_product, user_agent_ver keyword_params = getattr(secrets, 'GCE_KEYWORD_PARAMS', {}) if not project_id: project_id = keyword_params.get('project', None) + return (service_account_email, credentials_file, project_id) - # If we *still* don't have the credentials we need, then it's time to - # just fail out. - if service_account_email is None or credentials_file is None or project_id is None: - module.fail_json(msg='Missing GCE connection parameters in libcloud ' +def _get_gcp_credentials(module, require_valid_json=True, check_libcloud=False): + """ + Obtain GCP credentials by trying various methods. + + There are 3 ways to specify GCP credentials: + 1. Specify via Ansible module parameters (recommended). + 2. Specify via environment variables. Two sets of env vars are available: + a) GOOGLE_CLOUD_PROJECT, GOOGLE_CREDENTIALS_APPLICATION (preferred) + b) GCE_PROJECT, GCE_CREDENTIAL_FILE_PATH, GCE_EMAIL (legacy, not recommended; req'd if + using p12 key) + 3. Specify via libcloud secrets.py file (deprecated). + + There are 3 helper functions to assist in the above. + + Regardless of method, the user also has the option of specifying a JSON + file or a p12 file as the credentials file. JSON is strongly recommended and + p12 will be removed in the future. + + Additionally, flags may be set to require valid json and check the libcloud + version. + + :param module: initialized Ansible module object + :type module: `class AnsibleModule` + + :param require_valid_json: If true, require credentials to be valid JSON. Default is True. + :type require_valid_json: ``bool`` + + :params check_libcloud: If true, check the libcloud version available to see if + JSON creds are supported. + :type check_libcloud: ``bool`` + + :return: {'service_account_email': service_account_email, + 'credentials_file': credentials_file, + 'project_id': project_id} + :rtype: ``dict`` + """ + (service_account_email, + credentials_file, + project_id) = _get_gcp_ansible_credentials(module) + + # If any of the values are not given as parameters, check the appropriate + # environment variables. + (service_account_email, + credentials_file, + project_id) = _get_gcp_environment_credentials(service_account_email, + credentials_file, project_id) + + # If we still don't have one or more of our credentials, attempt to + # get the remaining values from the libcloud secrets file. + (service_account_email, + credentials_file, + project_id) = _get_gcp_libcloud_credentials(service_account_email, + credentials_file, project_id) + + if credentials_file is None or project_id is None or service_account_email is None: + if check_libcloud is True: + # TODO(supertom): this message is legacy and integration tests depend on it. + module.fail_json(msg='Missing GCE connection parameters in libcloud ' 'secrets file.') - return None - else: - # We have credentials but lets make sure that if they are JSON we have the minimum - # libcloud requirement met - try: - # Try to read credentials as JSON - with open(credentials_file) as credentials: - json.loads(credentials.read()) + return None + else: + if credentials_file is None or project_id is None: + module.fail_json(msg=('GCP connection error: enable to determine project (%s) or' + 'credentials file (%s)' % (project_id, credentials_file))) + + # ensure the credentials file is found and is in the proper format. + _validate_credentials_file(module, credentials_file, + require_valid_json=require_valid_json, + check_libcloud=check_libcloud) + + return {'service_account_email': service_account_email, + 'credentials_file': credentials_file, + 'project_id': project_id} + +def _validate_credentials_file(module, credentials_file, require_valid_json=True, check_libcloud=False): + """ + Check for valid credentials file. + + Optionally check for JSON format and if libcloud supports JSON. + + :param module: initialized Ansible module object + :type module: `class AnsibleModule` + + :param credentials_file: path to file on disk + :type credentials_file: ``str``. Complete path to file on disk. + + :param require_valid_json: If true, require credentials to be valid JSON. Default is True. + :type require_valid_json: ``bool`` + + :params check_libcloud: If true, check the libcloud version available to see if + JSON creds are supported. + :type check_libcloud: ``bool`` + + :returns: True + :rtype: ``bool`` + """ + try: + # Try to read credentials as JSON + with open(credentials_file) as credentials: + json.loads(credentials.read()) # If the credentials are proper JSON and we do not have the minimum # required libcloud version, bail out and return a descriptive error - if LooseVersion(libcloud.__version__) < '0.17.0': + if check_libcloud and LooseVersion(libcloud.__version__) < '0.17.0': module.fail_json(msg='Using JSON credentials but libcloud minimum version not met. ' 'Upgrade to libcloud>=0.17.0.') - return None - except ValueError as e: - # Not JSON - pass + return True + except IOError as e: + module.fail_json(msg='GCP Credentials File %s not found.', changed=False) + return False + except ValueError as e: + if require_valid_json: + module.fail_json(msg='GCP Credentials File %s invalid. Must be valid JSON.', changed=False) + else: + display.deprecated(msg=("Non-JSON credentials file provided. This format is deprecated. " + " Please generate a new JSON key from the Google Cloud console"), + version=2.5) + return True + +def gcp_connect(module, provider, get_driver, user_agent_product, user_agent_version): + """Return a Google libcloud driver connection.""" + if not HAS_LIBCLOUD_BASE: + module.fail_json(msg='libcloud must be installed to use this module') + + creds = _get_gcp_credentials(module, + require_valid_json=False, + check_libcloud=True) try: - gcp = get_driver(provider)(service_account_email, credentials_file, + gcp = get_driver(provider)(creds['service_account_email'], creds['credentials_file'], datacenter=module.params.get('zone', None), - project=project_id) + project=creds['project_id']) gcp.connection.user_agent_append("%s/%s" % ( user_agent_product, user_agent_version)) except (RuntimeError, ValueError) as e: @@ -112,6 +273,98 @@ def gcp_connect(module, provider, get_driver, user_agent_product, user_agent_ver return gcp + +def get_google_cloud_credentials(module, scopes=[]): + """ + Get credentials object for use with Google Cloud client. + + To connect via libcloud, don't use this function, use gcp_connect instead. For + Google Python API Client, see get_google_api_auth for how to connect. + + For more information on Google's client library options for Python, see: + U(https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries) + + Google Cloud example: + creds, params = get_google_cloud_credentials(module, scopes, user_agent_product, user_agent_version) + pubsub_client = pubsub.Client(project=params['project_id'], credentials=creds) + pubsub_client.user_agent = 'ansible-pubsub-0.1' + ... + + :param module: initialized Ansible module object + :type module: `class AnsibleModule` + + :param scopes: list of scopes + :type module: ``list`` of URIs + + :returns: A tuple containing (google authorized) credentials object and + params dict {'service_account_email': '...', 'credentials_file': '...', 'project_id': ...} + :rtype: ``tuple`` + """ + creds = _get_gcp_credentials(module, + require_valid_json=True, + check_libcloud=False) + try: + credentials = ServiceAccountCredentials.from_json_keyfile_name( + creds['credentials_file'], + scopes=scopes + ) + + return (credentials, creds) + except Exception as e: + module.fail_json(msg=unexpected_error_msg(e), changed=False) + return (None, None) + +def get_google_api_auth(module, scopes=[], user_agent_product='ansible-python-api', user_agent_version='NA'): + """ + Authentication for use with google-python-api-client. + + Function calls _get_gcp_credentials, which attempts to assemble the credentials from various locations. + Next it attempts to authenticate with Google. + + This function returns an httplib2 object that can be provided to the Google Python API client. + + For libcloud, don't use this function, use gcp_connect instead. For Google Cloud, See + get_google_cloud_credentials for how to connect. + + For more information on Google's client library options for Python, see: + U(https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries) + + Google API example: + http_auth, conn_params = gcp_api_auth(module, scopes, user_agent_product, user_agent_version) + service = build('myservice', 'v1', http=http_auth) + ... + + :param module: initialized Ansible module object + :type module: `class AnsibleModule` + + :param scopes: list of scopes + :type scopes: ``list`` of URIs + + :param user_agent_product: User agent product. eg: 'ansible-python-api' + :type user_agent_product: ``str`` + + :param user_agent_version: Version string to append to product. eg: 'NA' or '0.1' + :type user_agent_version: ``str`` + + :returns: A tuple containing (google authorized) httplib2 request object and a + params dict {'service_account_email': '...', 'credentials_file': '...', 'project_id': ...} + :rtype: ``tuple`` + """ + if not HAS_GOOGLE_API_LIB: + module.fail_json(msg="Please install google-api-python-client library") + # TODO(supertom): verify scopes + if not scopes: + scopes = ['https://www.googleapis.com/auth/cloud-platform'] + try: + (credentials, conn_params) = get_google_credentials(module, scopes, + require_valid_json=True, check_libcloud=False) + http = set_user_agent(Http(), '%s-%s' % (user_agent_product, user_agent_version)) + http_auth = credentials.authorize(http) + return (http_auth, conn_params) + except Exception as e: + module.fail_json(msg=unexpected_error_msg(e), changed=False) + return (None, None) + def unexpected_error_msg(error): """Create an error string based on passed in error.""" return 'Unexpected response: (%s). Detail: %s' % (str(error), traceback.format_exc(error)) diff --git a/test/units/module_utils/gcp/test_auth.py b/test/units/module_utils/gcp/test_auth.py new file mode 100644 index 0000000000..102a864218 --- /dev/null +++ b/test/units/module_utils/gcp/test_auth.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# (c) 2016, Tom Melendez +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +import os +import sys + +from ansible.compat.tests import mock, unittest +from ansible.module_utils.gcp import (_get_gcp_ansible_credentials, _get_gcp_environ_var, + _get_gcp_libcloud_credentials, _get_gcp_environment_credentials, + _validate_credentials_file) + +# Fake data/function used for testing +fake_env_data = {'GCE_EMAIL': 'gce-email'} +def fake_get_gcp_environ_var(var_name, default_value): + if var_name not in fake_env_data: + return default_value + else: + return fake_env_data[var_name] + +class GCPAuthTestCase(unittest.TestCase): + """Tests to verify different Auth mechanisms.""" + def test_get_gcp_ansible_credentials(self): + # create a fake (AnsibleModule) object to pass to the function + class FakeModule(object): + class Params(): + data = {} + def get(self, key, alt=None): + if key in self.data: + return self.data[key] + else: + return alt + def __init__(self, data={}): + self.params = FakeModule.Params() + self.params.data = data + input_data = {'service_account_email': 'mysa', + 'credentials_file': 'path-to-file.json', + 'project_id': 'my-cool-project'} + + module = FakeModule(input_data) + actual = _get_gcp_ansible_credentials(module) + expected = tuple(input_data.values()) + self.assertEqual(sorted(expected), sorted(actual)) + + def test_get_gcp_environ_var(self): + # Chose not to mock this so we could really verify that it + # works as expected. + existing_var_name = 'gcp_ansible_auth_test_54321' + non_existing_var_name = 'doesnt_exist_gcp_ansible_auth_test_12345' + os.environ[existing_var_name] = 'foobar' + self.assertEqual('foobar', _get_gcp_environ_var(existing_var_name, None)) + del os.environ[existing_var_name] + self.assertEqual('default_value', _get_gcp_environ_var( + non_existing_var_name, 'default_value')) + + def test_get_gcp_libcloud_credentials_no_import(self): + """No secrets imported. Whatever is sent in should come out.""" + actual = _get_gcp_libcloud_credentials(service_account_email=None, + credentials_file=None, + project_id=None) + expected = (None, None, None) + self.assertEqual(expected, actual) + # no libcloud, with values + actual = _get_gcp_libcloud_credentials(service_account_email='sa-email', + credentials_file='creds-file', + project_id='proj-id') + expected = ('sa-email', 'creds-file', 'proj-id') + self.assertEqual(expected, actual) + + @mock.patch("ansible.utils.display.Display.deprecated", + name='mock_deprecated', return_value=None) + def test_get_gcp_libcloud_credentials_import(self, mock_deprecated): + """secrets is imported and those values should be used.""" + # Note: Opted for a real class here rather than MagicMock as + # __getitem__ comes for free. + class FakeSecrets: + def __init__(self): + # 2 element list, service account email and creds file + self.GCE_PARAMS = ['secrets-sa', 'secrets-file.json'] + # dictionary with project_id, optionally auth_type + self.GCE_KEYWORD_PARAMS = {} + self.__file__ = 'THIS_IS_A_FAKEFILE_FOR_TESTING' + + # patch in module + fake_secrets = FakeSecrets() + patcher = mock.patch.dict(sys.modules,{'secrets': fake_secrets}) + patcher.start() + + # obtain sa and creds from secrets + actual = _get_gcp_libcloud_credentials(service_account_email=None, + credentials_file=None, + project_id='proj-id') + expected = ('secrets-sa', 'secrets-file.json', 'proj-id') + self.assertEqual(expected, actual) + + # fetch project id. Current logic requires sa-email or creds to be set. + fake_secrets.GCE_KEYWORD_PARAMS['project'] = 'new-proj-id' + fake_secrets.GCE_PARAMS[1] = 'my-creds.json' + + actual = _get_gcp_libcloud_credentials(service_account_email='my-sa', + credentials_file=None, + project_id=None) + expected = ('my-sa', 'my-creds.json', 'new-proj-id') + self.assertEqual(expected, actual) + + # stop patching + patcher.stop() + + @mock.patch("ansible.utils.display.Display.deprecated", + name='mock_deprecated', return_value=None) + def test_validate_credentials_file(self, mock_deprecated): + # TODO(supertom): Only dealing with p12 here, check the other states + # of this function + module = mock.MagicMock() + with mock.patch("ansible.module_utils.gcp.open", + mock.mock_open(read_data='foobar'), create=True) as m: + # pem condition, warning is surpressed with the return_value + credentials_file = '/foopath/pem.pem' + is_valid = _validate_credentials_file(module, + credentials_file=credentials_file, + require_valid_json=False, + check_libcloud=False) + self.assertTrue(is_valid) + + @mock.patch('ansible.module_utils.gcp._get_gcp_environ_var', + side_effect=fake_get_gcp_environ_var) + def test_get_gcp_environment_credentials(self, mockobj): + global fake_env_data + + actual = _get_gcp_environment_credentials(None, None, None) + expected = tuple(['gce-email', None, None]) + self.assertEqual(expected, actual) + + fake_env_data = {'GCE_PEM_FILE_PATH': '/path/to/pem.pem'} + expected = tuple([None, '/path/to/pem.pem', None]) + actual = _get_gcp_environment_credentials(None, None, None) + self.assertEqual(expected, actual) + + # pem and creds are set, expect creds + fake_env_data = {'GCE_PEM_FILE_PATH': '/path/to/pem.pem', + 'GCE_CREDENTIALS_FILE_PATH': '/path/to/creds.json'} + expected = tuple([None, '/path/to/creds.json', None]) + actual = _get_gcp_environment_credentials(None, None, None) + self.assertEqual(expected, actual) + + # expect GOOGLE_APPLICATION_CREDENTIALS over PEM + fake_env_data = {'GCE_PEM_FILE_PATH': '/path/to/pem.pem', + 'GOOGLE_APPLICATION_CREDENTIALS': '/path/to/appcreds.json'} + expected = tuple([None, '/path/to/appcreds.json', None]) + actual = _get_gcp_environment_credentials(None, None, None) + self.assertEqual(expected, actual) + + # project tests + fake_env_data = {'GCE_PROJECT': 'my-project'} + expected = tuple([None, None, 'my-project']) + actual = _get_gcp_environment_credentials(None, None, None) + self.assertEqual(expected, actual) + + fake_env_data = {'GOOGLE_CLOUD_PROJECT': 'my-cloud-project'} + expected = tuple([None, None, 'my-cloud-project']) + actual = _get_gcp_environment_credentials(None, None, None) + self.assertEqual(expected, actual) + + # data passed in, picking up project id only + fake_env_data = {'GOOGLE_CLOUD_PROJECT': 'my-project'} + expected = tuple(['my-sa-email', '/path/to/creds.json', 'my-project']) + actual = _get_gcp_environment_credentials('my-sa-email', '/path/to/creds.json', None) + self.assertEqual(expected, actual) + +if __name__ == '__main__': + unittest.main()