From 76eb024c250af3b7ff8adec909bef06667833509 Mon Sep 17 00:00:00 2001 From: Dave Costakos Date: Wed, 21 Jun 2023 16:24:14 -0700 Subject: [PATCH 1/4] Adding support for Google Secret Manager for issue 543 --- plugins/lookup/gcp_secret_manager.py | 213 +++++++++++++ plugins/modules/gcp_secret_manager.py | 413 ++++++++++++++++++++++++++ 2 files changed, 626 insertions(+) create mode 100644 plugins/lookup/gcp_secret_manager.py create mode 100644 plugins/modules/gcp_secret_manager.py diff --git a/plugins/lookup/gcp_secret_manager.py b/plugins/lookup/gcp_secret_manager.py new file mode 100644 index 0000000..56b11ac --- /dev/null +++ b/plugins/lookup/gcp_secret_manager.py @@ -0,0 +1,213 @@ +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + author: + - Dave Costakos + name: gcp_secret_manager + short_description: Get Secrets from Google Cloud as a Lookup plugin + description: + - retrieve secret keys in Secret Manager for use in playbooks + - see https://cloud.google.com/iam/docs/service-account-creds for details on creating + credentials for Google Cloud and the format of such credentials + - once a secret value is retreived, it is returned decoded. It is up to the developer + to maintain secrecy of this value once returned. + + options: + key: + description: + - the key of the secret to look up in Secret Manager + type: str + required: True + project: + description: + - The name of the google cloud project + - defaults to OS env variable GCP_PROJECT if not present + type: str + auth_kind: + description: + - the type of authentication to use with Google Cloud (i.e. serviceaccount or machineaccount) + - defaults to OS env variable GCP_AUTH_KIND if not present + type: str + version: + description: + - the version name of your secret to retrieve + type: str + default: latest + required: False + service_account_email: + description: + - email associated with the service account + - defaults to OS env variable GCP_SERVICE_ACCOUNT_EMAIL if not present + type: str + required: False + service_account_file: + description: + - JSON Credential file obtained from Google Cloud + - defaults to OS env variable GCP_SERVICE_ACCOUNT_FILE if not present + - see https://cloud.google.com/iam/docs/service-account-creds for details + type: str + required: False + service_account_info: + description: + - JSON Object representing the contents of a service_account_file obtained from Google Cloud + - defaults to OS env variable GCP_SERVICE_ACCOUNT_INFO if not present + type: jsonarg + required: False + errors: + description: + - how to handle errors + choices: ['strict','warn','ignore'] + default: strict +''' + +EXAMPLES = ''' +- name: Test secret using env variables for credentials + ansible.builtin.debug: + msg: "{{ lookup('google.cloud.gcp_secret_manager', key='secret_key') }}" + +- name: Test secret using explicit credentials + ansible.builtin.debug: + msg: "{{ lookup('google.cloud.gcp_secret_manager', key='secret_key', project='project', auth_kind='serviceaccount', service_account_file='file.json') }}" + +- name: Test getting specific version of a secret (old version) + ansible.builtin.debug: + msg: "{{ lookup('google.cloud.gcp_secret_manager', key='secret_key', version='1') }}" + +- name: Test getting specific version of a secret (new version) + ansible.builtin.debug: + msg: "{{ lookup('google.cloud.gcp_secret_manager', key='secret_key', version='2') }}" +''' + +RETURN = ''' + _raw: + description: the contents of the secret requested (please use "no_log" to not expose this secret) + type: list + elements: str +''' + +################################################################################ +# Imports +################################################################################ + +import json +import os +import base64 + + +from ansible.plugins.lookup import LookupBase + +try: + import requests + HAS_REQUESTS = True +except ImportError: + HAS_REQUESTS = False + +try: + import google.auth + from google.oauth2 import service_account + from google.auth.transport.requests import AuthorizedSession + HAS_GOOGLE_LIBRARIES = True +except ImportError: + HAS_GOOGLE_LIBRARIES = False + +from ansible_collections.google.cloud.plugins.module_utils.gcp_utils import GcpSession, GcpRequest +from ansible.errors import AnsibleError + +class GcpLookupException(Exception): + pass + +class LookupModule(LookupBase): + def run(self, terms, variables, **kwargs): + self.set_options(var_options=variables, direct=kwargs) + self.scopes = ["https://www.googleapis.com/auth/cloud-platform"] + self._validate() + self.service_acct_creds = self._credentials() + session = AuthorizedSession(self.service_acct_creds) + response = session.get("https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{key}/versions/{version}:access".format(**self.get_options())) + if response.status_code == 200: + result_data = response.json() + secret_value = base64.b64decode(result_data['payload']['data']) + return [ secret_value ] + else: + if self.get_option('errors') == 'warn': + self.warn(f"secret request returned bad status: {response.status_code} {response.json()}") + return [ '' ] + elif self.get_option('error') == 'ignore': + return [ '' ] + else: + raise AnsibleError(f"secret request returned bad status: {response.status_code} {response.json()}") + + def _validate(self): + if HAS_GOOGLE_LIBRARIES == False: + raise AnsibleError("Please install the google-auth library") + + if HAS_REQUESTS == False: + raise AnsibleError("Please install the requests library") + + if self.get_option('key') == None: + raise AnsibleError("'key' is a required parameter") + + if self.get_option('version') == None: + self.set_option('version', 'latest') + + self._set_from_env('project', 'GCP_PROJECT', True) + self._set_from_env('auth_kind', 'GCP_AUTH_KIND', True) + self._set_from_env('service_account_email', 'GCP_SERVICE_ACCOUNT_EMAIL') + self._set_from_env('service_account_file', 'GCP_SERVICE_ACCOUNT_FILE') + self._set_from_env('service_account_info', 'GCP_SERVICE_ACCOUNT_INFO') + + def _set_from_env(self, var=None, env_name=None, raise_on_empty=False): + if self.get_option(var) == None: + if env_name is not None and env_name in os.environ: + fallback = os.environ[env_name] + self.set_option(var, fallback) + + if self.get_option(var) == None and raise_on_empty: + msg = f"No key '{var}' provided" + if env_name is not None: + msg += f" and no fallback to env['{env_name}'] available" + raise AnsibleError(msg) + + def _credentials(self): + cred_type = self.get_option('auth_kind') + + if cred_type == 'application': + credentials, project_id = google.auth.default(scopes=self.scopes) + return credentials + + if cred_type == 'serviceaccount': + if self.get_option('service_account_file') is not None: + path = os.path.realpath(os.path.expanduser(self.get_option('service_account_file'))) + try: + svc_acct_creds = service_account.Credentials.from_service_account_file(path) + except OSError as e: + raise GcpLookupException("Unable to read service_account_file at %s: %s" % (path, e.strerror)) + + elif self.get_option('service_account_contents') is not None: + try: + info = json.loads(self.get_option('service_account_contents')) + except json.decoder.JSONDecodeError as e: + raise GcpLookupException("Unable to decode service_account_contents as JSON: %s" % e) + + svc_acct_creds = service_account.Credentials.from_service_account_info(info) + else: + raise GcpLookupException('Service Account authentication requires setting either service_account_file or service_account_contents') + + return svc_acct_creds.with_scopes(self.scopes) + + if cred_type == 'machineaccount': + self.svc_acct_creds = google.auth.compute_engine.Credentials(self.service_account_email) + return self.svc_acct_creds + + raise GcpLookupException("Credential type '%s' not implemented" % cred_type) + + + + + + + diff --git a/plugins/modules/gcp_secret_manager.py b/plugins/modules/gcp_secret_manager.py new file mode 100644 index 0000000..b8bf30b --- /dev/null +++ b/plugins/modules/gcp_secret_manager.py @@ -0,0 +1,413 @@ +#!/usr/bin/env python + +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +################################################################################ +# Documentation +################################################################################ + +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ["preview"], 'supported_by': 'community'} + +DOCUMENTATION=''' +--- +module: gcp_secret_manager +description: +- Simple create/delete of secrets +- add or delete secret versions +- other features like etags, replication, annontation expected to be managed outside of Ansible +requirements: +- python >= 2.6 +- requests >= 2.18.4 +- google-auth >= 1.3.0 +options: +options: + project: + description: + - The Google Cloud Platform project to use. + type: str + auth_kind: + description: + - The type of credential used. + type: str + required: true + choices: + - application + - machineaccount + - serviceaccount + service_account_contents: + description: + - The contents of a Service Account JSON file, either in a dictionary or as a + JSON string that represents it. + type: jsonarg + service_account_file: + description: + - The path of a Service Account JSON file if serviceaccount is selected as type. + type: path + service_account_email: + description: + - An optional service account email address if machineaccount is selected and + the user does not wish to use the default email. + type: str + scopes: + description: + - Array of scopes to be used + type: list + elements: str + name: + description: + - Name of the secret to be used + type: str + value: + description: + - The secret value that the secret should have + - this will be set upon create + - If the secret value is not this, a new version will be added with this value + type: str + state: + description: + - 'absent' or 'present': whether the secret should exist + type: str + return_value: + description: + - if true, the value of the secret will be returned unencrypted to Ansible + - if false, no value will be returned or decrypted + type: bool + default: true + version: + description: + - A version label to apply to the secret + - Default is "latest" which is the newest version of the secret + - "all" is also acceptable on delete (which will delete all versions of a secret) + type: str + default: 'latest' +''' + +EXAMPLES=''' +- name: Create a new secret + google.cloud.gcp_secret_manager: + name: secret_key + value: super_secret + state: present + auth_kind: serviceaccount + service_account_file: service_account_creds.json + +- name: Ensure the secretexists, fail otherwise and return the value + google.cloud.gcp_secret_manager: + name: secret_key + state: present + +- name: Ensure secret exists but don't return the value + google.cloud.gcp_secret_manager: + name: secret_key + state: present + return_value: false + +- name: Add a new version of a secret + google.cloud.gcp_secret_manager: + name: secret_key + value: updated super secret + state: present + +- name: Delete version 1 of a secret (but not the secret itself) + google.cloud.gcp_secret_manager: + name: secret_key + version: 1 + state: absent + +- name: Delete all versions of a secret + google.cloud.gcp_secret_manager: + name: secret_key + version: all + state: absent + +- name: Get +''' + +RETURN = ''' +resources: + description: List of resources + returned: always + type: complex + name: + description: + - The name of the secret + returned: success + type: str + version: + description: + - the version number of the secret returned + returned: success + type: str + url: + description: + - the Google Cloud URL used to make the request + returned: success + type: str + status_code: + description: + - the HTTP status code of the response to Google Cloud + returned: success + type: str + msg: + description: + - A message indicating what was done (or not done) + returned: success, failure + type: str + value: + description: + - The decrypted secret value, please use care with this + returned: success + type: str + payload: + description: + - The base 64 secret payload including CRC for validation + retunred: success + type: dict +''' + +################################################################################ +# Imports +################################################################################ + +from ansible_collections.google.cloud.plugins.module_utils.gcp_utils import ( + navigate_hash, + GcpSession, + GcpModule, + GcpRequest, + remove_nones_from_dict, + replace_resource_dict, +) + +import json +# for decoding and validating secrets +import base64 +import binascii + + +def get_auth(module): + return GcpSession(module, 'secret-manager') + +def self_access_link(module): + return "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions/{calc_version}:access".format(**module.params) + +def self_get_link(module): + return "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions/{calc_version}".format(**module.params) + +def self_update_link(module): + return "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions/{calc_version:version}".format(**module.params) + +def self_list_link(module): + return "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions?filter=state:ENABLED".format(**module.params) + +def self_delete_link(module): + return "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}".format(**module.params) + + +def fetch_resource(module, allow_not_found=True): + auth = get_auth(module) + # set version to the latest version because + # we can't be sure that "latest" is always going + # to be set if secret versions get disabled + # see https://issuetracker.google.com/issues/286489671 + if module.params['version'] == "latest" or module.params['version'] == 'all': + version_list = list_secret_versions(module) + latest_version = None + if version_list is None: + return None + + if "versions" in version_list: + latest_version = sorted(version_list['versions'], key=lambda d: d['name'])[-1]['name'].split('/')[-1] + module.params['calc_version'] = latest_version + else: + # if this occurs, there are no available secret versions + # handle the corner case that we tried to delete + # a secret version that doesn't exist + if module.params['state'] == "absent": + return { "action": "delete_secret" } + + link = self_access_link(module) + access_obj = return_if_object(module, auth.get(link), allow_not_found) + if access_obj is None: + return None + link = self_get_link(module) + get_obj = return_if_object(module, auth.get(link), allow_not_found) + if get_obj is None: + return None + return merge_dicts(get_obj, access_obj) + +def merge_dicts(x, y): + z = x.copy() + z.update(y) + return z + +def snake_to_camel(snake): + result = '' + capitalize_next = False + for char in snake: + if char == '_': + capitalize_next = True + else: + if capitalize_next: + result += char.upper() + capitalize_next = False + else: + result += char + return str(result) + +# create secret is a create call + an add version call +def create_secret(module): + # build the payload + payload = { "replication": { "automatic": {} } } + + url = "https://secretmanager.googleapis.com/v1/projects/{project}/secrets".format(**module.params) + auth = get_auth(module) + post_response = auth.post(url, body=payload, params={'secretId': module.params['name']}) + return update_secret(module) + +def update_secret(module): + # build the payload + b64_value = base64.b64encode(module.params['value'].encode("utf-8")).decode("utf-8") + payload = { + u'payload': { + u'data': b64_value + } + } + auth = get_auth(module) + url = "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}:addVersion".format(**module.params) + return return_if_object(module, auth.post(url, payload), False) + +def list_secret_versions(module): + # filter by only enabled secrets + url = self_list_link(module) + auth = get_auth(module) + return return_if_object(module, auth.get(url), True) + +# technically we're destroying the version +def delete_secret(module, destroy_all=False): + # delete secret does not take "latest" as a default version + # get the latest version if it doesn't exist in the request + version = module.params['version'] + auth = get_auth(module) + if version.lower() == "all" or destroy_all: + url = self_delete_link(module) + return return_if_object(module, auth.delete(url)) + else: + url = self_get_link(module) + ":destroy" + return return_if_object(module, auth.post(url, {}), False) + +def return_if_object(module, response, allow_not_found=False): + # If not found, return nothing. + if allow_not_found and response.status_code == 404: + return None + + if response.status_code == 409: + module.params['info'] = "exists already" + return None; + + # probably a code error + if response.status_code == 400: + module.fail_json(msg=f"unexpected REST failure: {response.json()['error']}") + + # If no content, return nothing. + if response.status_code == 204: + return None + + try: + module.raise_for_status(response) + result = response.json() + result['url'] = response.request.url + result['status_code'] = response.status_code + if "name" in result: + result['version'] = result['name'].split("/")[-1] + result['name'] = result['name'].split("/")[3] + + # base64 decode the value + if "payload" in result and "data" in result['payload']: + result['value'] = base64.b64decode(result['payload']['data']).decode("utf-8") + + except getattr(json.decoder, 'JSONDecodeError', ValueError): + module.fail_json(msg="Invalid JSON response with error: %s" % response.text) + + if navigate_hash(result, ['error', 'errors']): + module.fail_json(msg=navigate_hash(result, ['error', 'errors'])) + + return result + + +def main(): + # limited support for parameters described in the "Secret" resource + # in order to simplify and deploy primary use cases + # expectation is customers needing to support additional capabilities + # in the SecretPayload will do so outside of Ansible. + # ref: https://cloud.google.com/secret-manager/docs/reference/rest/v1/projects.secrets#Secret + module = GcpModule( + argument_spec=dict( + state=dict(default='present', choices=['present', 'absent'], type='str'), + name=dict(required=True, type='str', aliases=['key', 'secret']), + value=dict(required=False, type='str'), + version=dict(required=False, type='str', default='latest'), + return_value=dict(required=False, type='bool', default=True) + ) + ) + + if not module.params['scopes']: + module.params['scopes'] = ["https://www.googleapis.com/auth/cloud-platform"] + + module.params['calc_version'] = module.params['version'] + + state = module.params['state'] + fetch = fetch_resource(module, allow_not_found=True) + changed = False + + # nothing came back, so the secret doesn't exist + if not fetch: + # doesn't exist, must create + if module.params.get('value') and state == 'present': + # create a new secret + fetch = create_secret(module) + changed = True + # specified present but no value + # fail, let the user know + # that no secret could be created without a value to encrypt + elif state == 'present': + module.fail_json(msg="secret '{name}' not present in '{project}' and no value for the secret is provided".format(**module.params)), + + # secret is absent, success + else: + fetch = { "msg": "secret '{name}' in project '{project}' not present".format(**module.params)} + + else: + # delete the secret version (latest if no version is specified) + if state == "absent": + # delete the secret + fetch = delete_secret(module, ("action" in fetch)) + fetch['msg'] = "Secret Destroyed, it may take time to propagate" + changed = True + + # check to see if the values are the same, and update if neede + if "value" in fetch and module.params.get('value') is not None: + # Update secret + if fetch['value'] != module.params['value']: + update = update_secret(module) + changed = True + else: + fetch['msg'] = "values identical, no need to update secret" + + # pop value data if return_value == false + if module.params['return_value'] == False: + fetch.pop('value') + fetch.pop('payload') + if "msg" in fetch: + fetch['msg'] = f"{fetch['msg']} | not returning secret value since 'return_value is set to false" + else: + fetch['msg'] = "not returning secret value since 'return_value is set to false" + + fetch['changed'] = changed + fetch['name'] = module.params['name'] + + module.exit_json(**fetch) + + +if __name__ == "__main__": + main() + \ No newline at end of file From 953b06ff0599a080ccce0aefe9f112b09fd004e8 Mon Sep 17 00:00:00 2001 From: Dave Costakos Date: Thu, 22 Jun 2023 14:36:55 -0700 Subject: [PATCH 2/4] updated lookuup plugin based on comment https://github.com/ansible-collections/google.cloud/pull/578/files/76eb024c250af3b7ff8adec909bef06667833509# --- plugins/lookup/gcp_secret_manager.py | 215 +++++++++++++++----------- plugins/modules/gcp_secret_manager.py | 22 ++- 2 files changed, 146 insertions(+), 91 deletions(-) diff --git a/plugins/lookup/gcp_secret_manager.py b/plugins/lookup/gcp_secret_manager.py index 56b11ac..aa38da1 100644 --- a/plugins/lookup/gcp_secret_manager.py +++ b/plugins/lookup/gcp_secret_manager.py @@ -19,9 +19,13 @@ DOCUMENTATION = ''' options: key: description: - - the key of the secret to look up in Secret Manager + - the name of the secret to look up in Secret Manager type: str required: True + aliases: + - name + - secret + - secret_id project: description: - The name of the google cloud project @@ -57,11 +61,30 @@ DOCUMENTATION = ''' - defaults to OS env variable GCP_SERVICE_ACCOUNT_INFO if not present type: jsonarg required: False - errors: + access_token: + description: + - support for GCP Access Token + - defaults to OS env variable GCP_ACCESS_TOKEN if not present + type: str + required: False + on_error: description: - how to handle errors - choices: ['strict','warn','ignore'] - default: strict + - strict means raise an exception + - warn means warn, and return none + - ignore means just return none + type: str + required: False + choices: + - 'strict' + - 'warn' + - 'ignore' + default: 'strict' + scopes: + description: + - Authenticaiton scopes for Google Secret Manager + type: list + default: ["https://www.googleapis.com/auth/cloud-platform"] ''' EXAMPLES = ''' @@ -99,6 +122,8 @@ import base64 from ansible.plugins.lookup import LookupBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display try: import requests @@ -107,103 +132,117 @@ except ImportError: HAS_REQUESTS = False try: - import google.auth - from google.oauth2 import service_account - from google.auth.transport.requests import AuthorizedSession - HAS_GOOGLE_LIBRARIES = True + from ansible_collections.google.cloud.plugins.module_utils.gcp_utils import ( + GcpSession, + ) + HAS_GOOGLE_CLOUD_COLLECTION = True except ImportError: - HAS_GOOGLE_LIBRARIES = False + HAS_GOOGLE_CLOUD_COLLECTION = False -from ansible_collections.google.cloud.plugins.module_utils.gcp_utils import GcpSession, GcpRequest from ansible.errors import AnsibleError +from ansible.utils.display import Display class GcpLookupException(Exception): pass +class GcpMockModule(object): + def __init__(self, params): + self.params = params + + def fail_json(self, *args, **kwargs): + raise AnsibleError(kwargs["msg"]) + + def raise_for_status(self, response): + try: + response.raise_for_status() + except getattr(requests.exceptions, "RequestException"): + self.fail_json(msg="GCP returned error: %s" % response.json()) + class LookupModule(LookupBase): - def run(self, terms, variables, **kwargs): + def run(self, terms=None, variables=None, **kwargs): + self._display = Display() + if not HAS_GOOGLE_CLOUD_COLLECTION: + raise AnsibleError( + "gcp_secret lookup needs a supported version of the google.cloud collection installed. Use `ansible-galaxy collection install google.cloud` to install it" + ) self.set_options(var_options=variables, direct=kwargs) - self.scopes = ["https://www.googleapis.com/auth/cloud-platform"] - self._validate() - self.service_acct_creds = self._credentials() - session = AuthorizedSession(self.service_acct_creds) - response = session.get("https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{key}/versions/{version}:access".format(**self.get_options())) - if response.status_code == 200: - result_data = response.json() - secret_value = base64.b64decode(result_data['payload']['data']) - return [ secret_value ] + params = { + "key": self.get_option("key"), + "version": self.get_option("version"), + "access_token": self.get_option("access_token"), + "scopes": self.get_option("scopes"), + "on_error": self.get_option("on_error") + } + + params['name'] = params['key'] + + # support GCP_* env variables for some parameters + for param in ["project", "auth_kind", "service_account_file", "service_account_info", "service_account_email", "access_token"]: + params[param] = self.fallback_from_env(param) + + self._display.vvv(msg=f"Module Parameters: {params}") + fake_module = GcpMockModule(params) + result = self.get_secret(fake_module) + return [base64.b64decode(result)] + + def fallback_from_env(self, arg): + if self.get_option(arg): + return self.get_option(arg) else: - if self.get_option('errors') == 'warn': - self.warn(f"secret request returned bad status: {response.status_code} {response.json()}") - return [ '' ] - elif self.get_option('error') == 'ignore': - return [ '' ] - else: - raise AnsibleError(f"secret request returned bad status: {response.status_code} {response.json()}") + env_name = f"GCP_{arg.upper()}" + if env_name in os.environ: + self.set_option(arg, os.environ[env_name]) + return self.get_option(arg) + - def _validate(self): - if HAS_GOOGLE_LIBRARIES == False: - raise AnsibleError("Please install the google-auth library") - - if HAS_REQUESTS == False: - raise AnsibleError("Please install the requests library") - - if self.get_option('key') == None: - raise AnsibleError("'key' is a required parameter") - - if self.get_option('version') == None: - self.set_option('version', 'latest') + # set version to the latest version because + # we can't be sure that "latest" is always going + # to be set if secret versions get disabled + # see https://issuetracker.google.com/issues/286489671 + def get_latest_version(self, module, auth): + url = "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions?filter=state:ENABLED".format( + **module.params + ) + response = auth.get(url) + self._display.vvv(msg=f"List Version Response: {response.status_code} for {response.request.url}: {response.json()}") + if response.status_code != 200: + self.raise_error(module, f"unable to list versions of secret {response.status_code}") + version_list = response.json() + if "versions" in version_list: + return sorted(version_list['versions'], key=lambda d: d['name'])[-1]['name'].split('/')[-1] + else: + self.raise_error(module, f"Unable to list secret versions via {response.request.url}: {response.json()}") - self._set_from_env('project', 'GCP_PROJECT', True) - self._set_from_env('auth_kind', 'GCP_AUTH_KIND', True) - self._set_from_env('service_account_email', 'GCP_SERVICE_ACCOUNT_EMAIL') - self._set_from_env('service_account_file', 'GCP_SERVICE_ACCOUNT_FILE') - self._set_from_env('service_account_info', 'GCP_SERVICE_ACCOUNT_INFO') + + def raise_error(self, module, msg): + if module.params['on_error'] == 'strict': + raise GcpLookupException(msg) + elif module.params['on_error'] == 'warn': + self._display.warning(msg) + + return None + + def get_secret(self, module): + auth = GcpSession(module, "secretmanager") + if module.params['version'] == "latest": + module.params['calc_version'] = self.get_latest_version(module, auth) + else: + module.params['calc_version'] = module.params['version'] + + # there was an error listing secret versions + if module.params['calc_version'] is None: + return '' + + url = "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions/{calc_version}:access".format( + **module.params + ) + response = auth.get(url) + self._display.vvv(msg=f"Response: {response.status_code} for {response.request.url}: {response.json()}") + if response.status_code != 200: + self.raise_error(module, f"Failed to lookup secret value via {response.request.url} {response.status_code}") + return '' - def _set_from_env(self, var=None, env_name=None, raise_on_empty=False): - if self.get_option(var) == None: - if env_name is not None and env_name in os.environ: - fallback = os.environ[env_name] - self.set_option(var, fallback) - - if self.get_option(var) == None and raise_on_empty: - msg = f"No key '{var}' provided" - if env_name is not None: - msg += f" and no fallback to env['{env_name}'] available" - raise AnsibleError(msg) - - def _credentials(self): - cred_type = self.get_option('auth_kind') - - if cred_type == 'application': - credentials, project_id = google.auth.default(scopes=self.scopes) - return credentials - - if cred_type == 'serviceaccount': - if self.get_option('service_account_file') is not None: - path = os.path.realpath(os.path.expanduser(self.get_option('service_account_file'))) - try: - svc_acct_creds = service_account.Credentials.from_service_account_file(path) - except OSError as e: - raise GcpLookupException("Unable to read service_account_file at %s: %s" % (path, e.strerror)) - - elif self.get_option('service_account_contents') is not None: - try: - info = json.loads(self.get_option('service_account_contents')) - except json.decoder.JSONDecodeError as e: - raise GcpLookupException("Unable to decode service_account_contents as JSON: %s" % e) - - svc_acct_creds = service_account.Credentials.from_service_account_info(info) - else: - raise GcpLookupException('Service Account authentication requires setting either service_account_file or service_account_contents') - - return svc_acct_creds.with_scopes(self.scopes) - - if cred_type == 'machineaccount': - self.svc_acct_creds = google.auth.compute_engine.Credentials(self.service_account_email) - return self.svc_acct_creds - - raise GcpLookupException("Credential type '%s' not implemented" % cred_type) + return response.json()['payload']['data'] diff --git a/plugins/modules/gcp_secret_manager.py b/plugins/modules/gcp_secret_manager.py index b8bf30b..af5d5bd 100644 --- a/plugins/modules/gcp_secret_manager.py +++ b/plugins/modules/gcp_secret_manager.py @@ -57,6 +57,10 @@ options: description: - Name of the secret to be used type: str + aliases: + - key + - secret + - secret_id value: description: - The secret value that the secret should have @@ -80,6 +84,13 @@ options: - "all" is also acceptable on delete (which will delete all versions of a secret) type: str default: 'latest' + labels: + description: + - A set of key-value pairs to assign as labels to asecret + - only used in creation + - Note that the "value" piece of a label must contain only readable chars + type: dict + required: False ''' EXAMPLES=''' @@ -120,7 +131,6 @@ EXAMPLES=''' version: all state: absent -- name: Get ''' RETURN = ''' @@ -258,10 +268,14 @@ def snake_to_camel(snake): def create_secret(module): # build the payload payload = { "replication": { "automatic": {} } } + if module.params['labels']: + payload['labels'] = module.params['labels'] url = "https://secretmanager.googleapis.com/v1/projects/{project}/secrets".format(**module.params) auth = get_auth(module) post_response = auth.post(url, body=payload, params={'secretId': module.params['name']}) + # validate create + module.raise_for_status(post_response) return update_secret(module) def update_secret(module): @@ -343,13 +357,15 @@ def main(): module = GcpModule( argument_spec=dict( state=dict(default='present', choices=['present', 'absent'], type='str'), - name=dict(required=True, type='str', aliases=['key', 'secret']), + name=dict(required=True, type='str', aliases=['key', 'secret', 'secret_id']), value=dict(required=False, type='str'), version=dict(required=False, type='str', default='latest'), - return_value=dict(required=False, type='bool', default=True) + return_value=dict(required=False, type='bool', default=True), + labels=dict(required=False, type='dict', default=dict()) ) ) + if not module.params['scopes']: module.params['scopes'] = ["https://www.googleapis.com/auth/cloud-platform"] From 3ce29db3ee94835364e806cf0bcb48cea1b49d08 Mon Sep 17 00:00:00 2001 From: Dave Costakos Date: Fri, 14 Jul 2023 10:31:52 -0700 Subject: [PATCH 3/4] updated plugsins based on feedback, fixed linting and documentation errors. --- plugins/lookup/gcp_secret_manager.py | 43 +++++++++++----------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/plugins/lookup/gcp_secret_manager.py b/plugins/lookup/gcp_secret_manager.py index aa38da1..bd9ca55 100644 --- a/plugins/lookup/gcp_secret_manager.py +++ b/plugins/lookup/gcp_secret_manager.py @@ -14,7 +14,7 @@ DOCUMENTATION = ''' - see https://cloud.google.com/iam/docs/service-account-creds for details on creating credentials for Google Cloud and the format of such credentials - once a secret value is retreived, it is returned decoded. It is up to the developer - to maintain secrecy of this value once returned. + to maintain secrecy of this value once returned. options: key: @@ -62,7 +62,7 @@ DOCUMENTATION = ''' type: jsonarg required: False access_token: - description: + description: - support for GCP Access Token - defaults to OS env variable GCP_ACCESS_TOKEN if not present type: str @@ -116,11 +116,9 @@ RETURN = ''' # Imports ################################################################################ -import json import os import base64 - from ansible.plugins.lookup import LookupBase from ansible.errors import AnsibleError from ansible.utils.display import Display @@ -139,12 +137,11 @@ try: except ImportError: HAS_GOOGLE_CLOUD_COLLECTION = False -from ansible.errors import AnsibleError -from ansible.utils.display import Display class GcpLookupException(Exception): pass + class GcpMockModule(object): def __init__(self, params): self.params = params @@ -158,20 +155,23 @@ class GcpMockModule(object): except getattr(requests.exceptions, "RequestException"): self.fail_json(msg="GCP returned error: %s" % response.json()) + class LookupModule(LookupBase): def run(self, terms=None, variables=None, **kwargs): self._display = Display() if not HAS_GOOGLE_CLOUD_COLLECTION: raise AnsibleError( - "gcp_secret lookup needs a supported version of the google.cloud collection installed. Use `ansible-galaxy collection install google.cloud` to install it" - ) + """gcp_secret lookup needs a supported version of the google.cloud + collection installed. Use `ansible-galaxy collection install google.cloud` + to install it""" + ) self.set_options(var_options=variables, direct=kwargs) params = { - "key": self.get_option("key"), - "version": self.get_option("version"), + "key": self.get_option("key"), + "version": self.get_option("version"), "access_token": self.get_option("access_token"), - "scopes": self.get_option("scopes"), - "on_error": self.get_option("on_error") + "scopes": self.get_option("scopes"), + "on_error": self.get_option("on_error") } params['name'] = params['key'] @@ -184,7 +184,7 @@ class LookupModule(LookupBase): fake_module = GcpMockModule(params) result = self.get_secret(fake_module) return [base64.b64decode(result)] - + def fallback_from_env(self, arg): if self.get_option(arg): return self.get_option(arg) @@ -193,10 +193,9 @@ class LookupModule(LookupBase): if env_name in os.environ: self.set_option(arg, os.environ[env_name]) return self.get_option(arg) - # set version to the latest version because - # we can't be sure that "latest" is always going + # we can't be sure that "latest" is always going # to be set if secret versions get disabled # see https://issuetracker.google.com/issues/286489671 def get_latest_version(self, module, auth): @@ -213,15 +212,14 @@ class LookupModule(LookupBase): else: self.raise_error(module, f"Unable to list secret versions via {response.request.url}: {response.json()}") - def raise_error(self, module, msg): if module.params['on_error'] == 'strict': raise GcpLookupException(msg) elif module.params['on_error'] == 'warn': self._display.warning(msg) - + return None - + def get_secret(self, module): auth = GcpSession(module, "secretmanager") if module.params['version'] == "latest": @@ -241,12 +239,5 @@ class LookupModule(LookupBase): if response.status_code != 200: self.raise_error(module, f"Failed to lookup secret value via {response.request.url} {response.status_code}") return '' - + return response.json()['payload']['data'] - - - - - - - From 40d2c9a7d581105636d466e236180c8a5152e6c6 Mon Sep 17 00:00:00 2001 From: Dave Costakos Date: Fri, 14 Jul 2023 10:33:15 -0700 Subject: [PATCH 4/4] updated plugsins based on feedback, fixed linting and documentation errors. --- plugins/modules/gcp_secret_manager.py | 143 +++++++++++++++----------- 1 file changed, 84 insertions(+), 59 deletions(-) diff --git a/plugins/modules/gcp_secret_manager.py b/plugins/modules/gcp_secret_manager.py index af5d5bd..f8d5623 100644 --- a/plugins/modules/gcp_secret_manager.py +++ b/plugins/modules/gcp_secret_manager.py @@ -1,25 +1,35 @@ -#!/usr/bin/env python +#!/usr/bin/python -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt +# or https://www.gnu.org/licenses/gpl-3.0.txt) # SPDX-License-Identifier: GPL-3.0-or-later + ################################################################################ # Documentation ################################################################################ + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ["preview"], 'supported_by': 'community'} -DOCUMENTATION=''' +DOCUMENTATION = ''' --- module: gcp_secret_manager description: -- Simple create/delete of secrets -- add or delete secret versions -- other features like etags, replication, annontation expected to be managed outside of Ansible +- Access secrets stored in Google Secrets Manager. +- Create new secrets. +- Create new secret values. +- Add/remove versions of secrets. +- Please note that other features like etags, replication, annontation expected to be managed outside of Ansible. +short_description: Access and Update Google Cloud Secrets Manager objects +author: Dave Costakos @RedHat requirements: - python >= 2.6 - requests >= 2.18.4 - google-auth >= 1.3.0 -options: options: project: description: @@ -57,6 +67,7 @@ options: description: - Name of the secret to be used type: str + required: true aliases: - key - secret @@ -69,7 +80,11 @@ options: type: str state: description: - - 'absent' or 'present': whether the secret should exist + - whether the secret should exist + default: present + choices: + - absent + - present type: str return_value: description: @@ -81,19 +96,31 @@ options: description: - A version label to apply to the secret - Default is "latest" which is the newest version of the secret - - "all" is also acceptable on delete (which will delete all versions of a secret) + - The special "all" is also acceptable on delete (which will delete all versions of a secret) type: str - default: 'latest' + default: latest labels: description: - A set of key-value pairs to assign as labels to asecret - only used in creation - Note that the "value" piece of a label must contain only readable chars type: dict - required: False +notes: +- 'API Reference: U(https://cloud.google.com/secret-manager/docs/reference/rests)' +- 'Official Documentation: U(https://cloud.google.com/secret-manager/docs/overview)' +- for authentication, you can set service_account_file using the C(GCP_SERVICE_ACCOUNT_FILE) + env variable. +- for authentication, you can set service_account_contents using the C(GCP_SERVICE_ACCOUNT_CONTENTS) + env variable. +- For authentication, you can set service_account_email using the C(GCP_SERVICE_ACCOUNT_EMAIL) + env variable. +- For authentication, you can set auth_kind using the C(GCP_AUTH_KIND) env variable. +- For authentication, you can set scopes using the C(GCP_SCOPES) env variable. +- Environment variables values will only be used if the playbook values are not set. +- The I(service_account_email) and I(service_account_file) options are mutually exclusive. ''' -EXAMPLES=''' +EXAMPLES = r''' - name: Create a new secret google.cloud.gcp_secret_manager: name: secret_key @@ -110,7 +137,7 @@ EXAMPLES=''' - name: Ensure secret exists but don't return the value google.cloud.gcp_secret_manager: name: secret_key - state: present + state: present return_value: false - name: Add a new version of a secret @@ -131,9 +158,15 @@ EXAMPLES=''' version: all state: absent +- name: Create a secret with labels + google.cloud.gcp_secret_manager: + name: secret_key + value: super_secret + labels: + key_name: "ansible_rox" ''' -RETURN = ''' +RETURN = r''' resources: description: List of resources returned: always @@ -182,33 +215,35 @@ resources: from ansible_collections.google.cloud.plugins.module_utils.gcp_utils import ( navigate_hash, GcpSession, - GcpModule, - GcpRequest, - remove_nones_from_dict, - replace_resource_dict, + GcpModule ) -import json # for decoding and validating secrets +import json import base64 -import binascii +import copy def get_auth(module): return GcpSession(module, 'secret-manager') + def self_access_link(module): return "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions/{calc_version}:access".format(**module.params) + def self_get_link(module): return "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions/{calc_version}".format(**module.params) + def self_update_link(module): return "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions/{calc_version:version}".format(**module.params) + def self_list_link(module): return "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions?filter=state:ENABLED".format(**module.params) + def self_delete_link(module): return "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}".format(**module.params) @@ -216,7 +251,7 @@ def self_delete_link(module): def fetch_resource(module, allow_not_found=True): auth = get_auth(module) # set version to the latest version because - # we can't be sure that "latest" is always going + # we can't be sure that "latest" is always going # to be set if secret versions get disabled # see https://issuetracker.google.com/issues/286489671 if module.params['version'] == "latest" or module.params['version'] == 'all': @@ -224,7 +259,7 @@ def fetch_resource(module, allow_not_found=True): latest_version = None if version_list is None: return None - + if "versions" in version_list: latest_version = sorted(version_list['versions'], key=lambda d: d['name'])[-1]['name'].split('/')[-1] module.params['calc_version'] = latest_version @@ -233,7 +268,7 @@ def fetch_resource(module, allow_not_found=True): # handle the corner case that we tried to delete # a secret version that doesn't exist if module.params['state'] == "absent": - return { "action": "delete_secret" } + return {"action": "delete_secret"} link = self_access_link(module) access_obj = return_if_object(module, auth.get(link), allow_not_found) @@ -245,29 +280,17 @@ def fetch_resource(module, allow_not_found=True): return None return merge_dicts(get_obj, access_obj) + def merge_dicts(x, y): - z = x.copy() + z = copy.deepcopy(x) z.update(y) return z -def snake_to_camel(snake): - result = '' - capitalize_next = False - for char in snake: - if char == '_': - capitalize_next = True - else: - if capitalize_next: - result += char.upper() - capitalize_next = False - else: - result += char - return str(result) # create secret is a create call + an add version call def create_secret(module): # build the payload - payload = { "replication": { "automatic": {} } } + payload = {"replication": {"automatic": {}}} if module.params['labels']: payload['labels'] = module.params['labels'] @@ -278,6 +301,7 @@ def create_secret(module): module.raise_for_status(post_response) return update_secret(module) + def update_secret(module): # build the payload b64_value = base64.b64encode(module.params['value'].encode("utf-8")).decode("utf-8") @@ -290,12 +314,14 @@ def update_secret(module): url = "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}:addVersion".format(**module.params) return return_if_object(module, auth.post(url, payload), False) + def list_secret_versions(module): # filter by only enabled secrets url = self_list_link(module) auth = get_auth(module) return return_if_object(module, auth.get(url), True) + # technically we're destroying the version def delete_secret(module, destroy_all=False): # delete secret does not take "latest" as a default version @@ -303,24 +329,25 @@ def delete_secret(module, destroy_all=False): version = module.params['version'] auth = get_auth(module) if version.lower() == "all" or destroy_all: - url = self_delete_link(module) - return return_if_object(module, auth.delete(url)) + url = self_delete_link(module) + return return_if_object(module, auth.delete(url)) else: - url = self_get_link(module) + ":destroy" - return return_if_object(module, auth.post(url, {}), False) + url = self_get_link(module) + ":destroy" + return return_if_object(module, auth.post(url, {}), False) + def return_if_object(module, response, allow_not_found=False): # If not found, return nothing. if allow_not_found and response.status_code == 404: return None - + if response.status_code == 409: module.params['info'] = "exists already" - return None; + return None # probably a code error if response.status_code == 400: - module.fail_json(msg=f"unexpected REST failure: {response.json()['error']}") + module.fail_json(msg="unexpected REST failure: %s" % response.json()['error']) # If no content, return nothing. if response.status_code == 204: @@ -334,11 +361,11 @@ def return_if_object(module, response, allow_not_found=False): if "name" in result: result['version'] = result['name'].split("/")[-1] result['name'] = result['name'].split("/")[3] - + # base64 decode the value if "payload" in result and "data" in result['payload']: result['value'] = base64.b64decode(result['payload']['data']).decode("utf-8") - + except getattr(json.decoder, 'JSONDecodeError', ValueError): module.fail_json(msg="Invalid JSON response with error: %s" % response.text) @@ -351,7 +378,7 @@ def return_if_object(module, response, allow_not_found=False): def main(): # limited support for parameters described in the "Secret" resource # in order to simplify and deploy primary use cases - # expectation is customers needing to support additional capabilities + # expectation is customers needing to support additional capabilities # in the SecretPayload will do so outside of Ansible. # ref: https://cloud.google.com/secret-manager/docs/reference/rest/v1/projects.secrets#Secret module = GcpModule( @@ -365,7 +392,6 @@ def main(): ) ) - if not module.params['scopes']: module.params['scopes'] = ["https://www.googleapis.com/auth/cloud-platform"] @@ -386,11 +412,11 @@ def main(): # fail, let the user know # that no secret could be created without a value to encrypt elif state == 'present': - module.fail_json(msg="secret '{name}' not present in '{project}' and no value for the secret is provided".format(**module.params)), + module.fail_json(msg="secret '{name}' not present in '{project}' and no value for the secret is provided".format(**module.params)) # secret is absent, success else: - fetch = { "msg": "secret '{name}' in project '{project}' not present".format(**module.params)} + fetch = {"msg": "secret '{name}' in project '{project}' not present".format(**module.params)} else: # delete the secret version (latest if no version is specified) @@ -404,26 +430,25 @@ def main(): if "value" in fetch and module.params.get('value') is not None: # Update secret if fetch['value'] != module.params['value']: - update = update_secret(module) + update_secret(module) changed = True else: fetch['msg'] = "values identical, no need to update secret" - - # pop value data if return_value == false - if module.params['return_value'] == False: + + # pop value data if return_value == false + if module.params['return_value'] is False: fetch.pop('value') fetch.pop('payload') if "msg" in fetch: fetch['msg'] = f"{fetch['msg']} | not returning secret value since 'return_value is set to false" else: fetch['msg'] = "not returning secret value since 'return_value is set to false" - + fetch['changed'] = changed fetch['name'] = module.params['name'] module.exit_json(**fetch) - - + + if __name__ == "__main__": main() - \ No newline at end of file