From 459190f3948e7172dce14ce16fc28484ca32e571 Mon Sep 17 00:00:00 2001 From: The Magician Date: Wed, 17 Jul 2019 11:09:57 -0700 Subject: [PATCH] ML Engine: Models in Ansible (#313) Signed-off-by: Modular Magician --- plugins/modules/gcp_mlengine_model.py | 342 ++++++++++++++++++ plugins/modules/gcp_mlengine_model_facts.py | 165 +++++++++ tests/integration/gcp_mlengine_model/aliases | 2 + .../gcp_mlengine_model/defaults/main.yml | 2 + .../gcp_mlengine_model/meta/main.yml | 0 .../gcp_mlengine_model/tasks/main.yml | 113 ++++++ 6 files changed, 624 insertions(+) create mode 100644 plugins/modules/gcp_mlengine_model.py create mode 100644 plugins/modules/gcp_mlengine_model_facts.py create mode 100644 tests/integration/gcp_mlengine_model/aliases create mode 100644 tests/integration/gcp_mlengine_model/defaults/main.yml create mode 100644 tests/integration/gcp_mlengine_model/meta/main.yml create mode 100644 tests/integration/gcp_mlengine_model/tasks/main.yml diff --git a/plugins/modules/gcp_mlengine_model.py b/plugins/modules/gcp_mlengine_model.py new file mode 100644 index 0000000..f370667 --- /dev/null +++ b/plugins/modules/gcp_mlengine_model.py @@ -0,0 +1,342 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017 Google +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# ---------------------------------------------------------------------------- +# +# *** AUTO GENERATED CODE *** AUTO GENERATED CODE *** +# +# ---------------------------------------------------------------------------- +# +# This file is automatically generated by Magic Modules and manual +# changes will be clobbered when the file is regenerated. +# +# Please read more about how to change this file at +# https://www.github.com/GoogleCloudPlatform/magic-modules +# +# ---------------------------------------------------------------------------- + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +################################################################################ +# Documentation +################################################################################ + +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ["preview"], 'supported_by': 'community'} + +DOCUMENTATION = ''' +--- +module: gcp_mlengine_model +description: +- Represents a machine learning solution. +- A model can have multiple versions, each of which is a deployed, trained model ready + to receive prediction requests. The model itself is just a container. +short_description: Creates a GCP Model +version_added: 2.9 +author: Google Inc. (@googlecloudplatform) +requirements: +- python >= 2.6 +- requests >= 2.18.4 +- google-auth >= 1.3.0 +options: + state: + description: + - Whether the given object should exist in GCP + choices: + - present + - absent + default: present + name: + description: + - The name specified for the model. + required: true + type: str + description: + description: + - The description specified for the model when it was created. + required: false + type: str + default_version: + description: + - The default version of the model. This version will be used to handle prediction + requests that do not specify a version. + required: false + type: dict + suboptions: + name: + description: + - The name specified for the version when it was created. + required: false + type: str + regions: + description: + - The list of regions where the model is going to be deployed. + - Currently only one region per model is supported . + required: false + type: list + online_prediction_logging: + description: + - If true, online prediction access logs are sent to StackDriver Logging. + required: false + type: bool + online_prediction_console_logging: + description: + - If true, online prediction nodes send stderr and stdout streams to Stackdriver + Logging. + required: false + type: bool + labels: + description: + - One or more labels that you can add, to organize your models. + required: false + type: dict +extends_documentation_fragment: gcp +''' + +EXAMPLES = ''' +- name: create a model + gcp_mlengine_model: + name: "{{ resource_name | replace('-', '_') }}" + description: My model + regions: + - us-central1 + project: test_project + auth_kind: serviceaccount + service_account_file: "/tmp/auth.pem" + state: present +''' + +RETURN = ''' +name: + description: + - The name specified for the model. + returned: success + type: str +description: + description: + - The description specified for the model when it was created. + returned: success + type: str +defaultVersion: + description: + - The default version of the model. This version will be used to handle prediction + requests that do not specify a version. + returned: success + type: complex + contains: + name: + description: + - The name specified for the version when it was created. + returned: success + type: str +regions: + description: + - The list of regions where the model is going to be deployed. + - Currently only one region per model is supported . + returned: success + type: list +onlinePredictionLogging: + description: + - If true, online prediction access logs are sent to StackDriver Logging. + returned: success + type: bool +onlinePredictionConsoleLogging: + description: + - If true, online prediction nodes send stderr and stdout streams to Stackdriver + Logging. + returned: success + type: bool +labels: + description: + - One or more labels that you can add, to organize your models. + returned: success + type: dict +''' + +################################################################################ +# Imports +################################################################################ + +from ansible.module_utils.gcp_utils import navigate_hash, GcpSession, GcpModule, GcpRequest, remove_nones_from_dict, replace_resource_dict +import json + +################################################################################ +# Main +################################################################################ + + +def main(): + """Main function""" + + module = GcpModule( + argument_spec=dict( + state=dict(default='present', choices=['present', 'absent'], type='str'), + name=dict(required=True, type='str'), + description=dict(type='str'), + default_version=dict(type='dict', options=dict(name=dict(type='str'))), + regions=dict(type='list', elements='str'), + online_prediction_logging=dict(type='bool'), + online_prediction_console_logging=dict(type='bool'), + labels=dict(type='dict'), + ) + ) + + if not module.params['scopes']: + module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform'] + + state = module.params['state'] + + fetch = fetch_resource(module, self_link(module)) + changed = False + + if fetch: + if state == 'present': + if is_different(module, fetch): + update(module, self_link(module)) + fetch = fetch_resource(module, self_link(module)) + changed = True + else: + delete(module, self_link(module)) + fetch = {} + changed = True + else: + if state == 'present': + fetch = create(module, collection(module)) + changed = True + else: + fetch = {} + + fetch.update({'changed': changed}) + + module.exit_json(**fetch) + + +def create(module, link): + auth = GcpSession(module, 'mlengine') + return return_if_object(module, auth.post(link, resource_to_request(module))) + + +def update(module, link): + delete(module, self_link(module)) + create(module, collection(module)) + + +def delete(module, link): + auth = GcpSession(module, 'mlengine') + return return_if_object(module, auth.delete(link)) + + +def resource_to_request(module): + request = { + u'name': module.params.get('name'), + u'description': module.params.get('description'), + u'defaultVersion': ModelDefaultversion(module.params.get('default_version', {}), module).to_request(), + u'regions': module.params.get('regions'), + u'onlinePredictionLogging': module.params.get('online_prediction_logging'), + u'onlinePredictionConsoleLogging': module.params.get('online_prediction_console_logging'), + u'labels': module.params.get('labels'), + } + return_vals = {} + for k, v in request.items(): + if v or v is False: + return_vals[k] = v + + return return_vals + + +def fetch_resource(module, link, allow_not_found=True): + auth = GcpSession(module, 'mlengine') + return return_if_object(module, auth.get(link), allow_not_found) + + +def self_link(module): + return "https://ml.googleapis.com/v1/projects/{project}/models/{name}".format(**module.params) + + +def collection(module): + return "https://ml.googleapis.com/v1/projects/{project}/models".format(**module.params) + + +def return_if_object(module, response, allow_not_found=False): + # If not found, return nothing. + if allow_not_found and response.status_code == 404: + return None + + # If no content, return nothing. + if response.status_code == 204: + return None + + try: + module.raise_for_status(response) + result = response.json() + except getattr(json.decoder, 'JSONDecodeError', ValueError): + module.fail_json(msg="Invalid JSON response with error: %s" % response.text) + + result = decode_response(result, module) + + if navigate_hash(result, ['error', 'errors']): + module.fail_json(msg=navigate_hash(result, ['error', 'errors'])) + + return result + + +def is_different(module, response): + request = resource_to_request(module) + response = response_to_hash(module, response) + request = decode_response(request, module) + + # Remove all output-only from response. + response_vals = {} + for k, v in response.items(): + if k in request: + response_vals[k] = v + + request_vals = {} + for k, v in request.items(): + if k in response: + request_vals[k] = v + + return GcpRequest(request_vals) != GcpRequest(response_vals) + + +# Remove unnecessary properties from the response. +# This is for doing comparisons with Ansible's current parameters. +def response_to_hash(module, response): + return { + u'name': response.get(u'name'), + u'description': response.get(u'description'), + u'defaultVersion': ModelDefaultversion(response.get(u'defaultVersion', {}), module).from_response(), + u'regions': response.get(u'regions'), + u'onlinePredictionLogging': response.get(u'onlinePredictionLogging'), + u'onlinePredictionConsoleLogging': response.get(u'onlinePredictionConsoleLogging'), + u'labels': response.get(u'labels'), + } + + +# Short names are given (and expected) by the API +# but are returned as full names. +def decode_response(response, module): + response['name'] = response['name'].split('/')[-1] + return response + + +class ModelDefaultversion(object): + def __init__(self, request, module): + self.module = module + if request: + self.request = request + else: + self.request = {} + + def to_request(self): + return remove_nones_from_dict({u'name': self.request.get('name')}) + + def from_response(self): + return remove_nones_from_dict({u'name': self.request.get(u'name')}) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/gcp_mlengine_model_facts.py b/plugins/modules/gcp_mlengine_model_facts.py new file mode 100644 index 0000000..1e013b3 --- /dev/null +++ b/plugins/modules/gcp_mlengine_model_facts.py @@ -0,0 +1,165 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017 Google +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# ---------------------------------------------------------------------------- +# +# *** AUTO GENERATED CODE *** AUTO GENERATED CODE *** +# +# ---------------------------------------------------------------------------- +# +# This file is automatically generated by Magic Modules and manual +# changes will be clobbered when the file is regenerated. +# +# Please read more about how to change this file at +# https://www.github.com/GoogleCloudPlatform/magic-modules +# +# ---------------------------------------------------------------------------- + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +################################################################################ +# Documentation +################################################################################ + +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ["preview"], 'supported_by': 'community'} + +DOCUMENTATION = ''' +--- +module: gcp_mlengine_model_facts +description: +- Gather facts for GCP Model +short_description: Gather facts for GCP Model +version_added: 2.9 +author: Google Inc. (@googlecloudplatform) +requirements: +- python >= 2.6 +- requests >= 2.18.4 +- google-auth >= 1.3.0 +options: {} +extends_documentation_fragment: gcp +''' + +EXAMPLES = ''' +- name: " a model facts" + gcp_mlengine_model_facts: + project: test_project + auth_kind: serviceaccount + service_account_file: "/tmp/auth.pem" + state: facts +''' + +RETURN = ''' +resources: + description: List of resources + returned: always + type: complex + contains: + name: + description: + - The name specified for the model. + returned: success + type: str + description: + description: + - The description specified for the model when it was created. + returned: success + type: str + defaultVersion: + description: + - The default version of the model. This version will be used to handle prediction + requests that do not specify a version. + returned: success + type: complex + contains: + name: + description: + - The name specified for the version when it was created. + returned: success + type: str + regions: + description: + - The list of regions where the model is going to be deployed. + - Currently only one region per model is supported . + returned: success + type: list + onlinePredictionLogging: + description: + - If true, online prediction access logs are sent to StackDriver Logging. + returned: success + type: bool + onlinePredictionConsoleLogging: + description: + - If true, online prediction nodes send stderr and stdout streams to Stackdriver + Logging. + returned: success + type: bool + labels: + description: + - One or more labels that you can add, to organize your models. + returned: success + type: dict +''' + +################################################################################ +# Imports +################################################################################ +from ansible.module_utils.gcp_utils import navigate_hash, GcpSession, GcpModule, GcpRequest +import json + +################################################################################ +# Main +################################################################################ + + +def main(): + module = GcpModule(argument_spec=dict()) + + if not module.params['scopes']: + module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform'] + + items = fetch_list(module, collection(module)) + if items.get('models'): + items = items.get('models') + else: + items = [] + return_value = {'resources': items} + module.exit_json(**return_value) + + +def collection(module): + return "https://ml.googleapis.com/v1/projects/{project}/models".format(**module.params) + + +def fetch_list(module, link): + auth = GcpSession(module, 'mlengine') + response = auth.get(link) + return return_if_object(module, response) + + +def return_if_object(module, response): + # If not found, return nothing. + if response.status_code == 404: + return None + + # If no content, return nothing. + if response.status_code == 204: + return None + + try: + module.raise_for_status(response) + result = response.json() + except getattr(json.decoder, 'JSONDecodeError', ValueError) as inst: + module.fail_json(msg="Invalid JSON response with error: %s" % inst) + + if navigate_hash(result, ['error', 'errors']): + module.fail_json(msg=navigate_hash(result, ['error', 'errors'])) + + return result + + +if __name__ == "__main__": + main() diff --git a/tests/integration/gcp_mlengine_model/aliases b/tests/integration/gcp_mlengine_model/aliases new file mode 100644 index 0000000..9812f01 --- /dev/null +++ b/tests/integration/gcp_mlengine_model/aliases @@ -0,0 +1,2 @@ +cloud/gcp +unsupported diff --git a/tests/integration/gcp_mlengine_model/defaults/main.yml b/tests/integration/gcp_mlengine_model/defaults/main.yml new file mode 100644 index 0000000..ba66644 --- /dev/null +++ b/tests/integration/gcp_mlengine_model/defaults/main.yml @@ -0,0 +1,2 @@ +--- +resource_name: "{{ resource_prefix }}" diff --git a/tests/integration/gcp_mlengine_model/meta/main.yml b/tests/integration/gcp_mlengine_model/meta/main.yml new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/gcp_mlengine_model/tasks/main.yml b/tests/integration/gcp_mlengine_model/tasks/main.yml new file mode 100644 index 0000000..50eec5a --- /dev/null +++ b/tests/integration/gcp_mlengine_model/tasks/main.yml @@ -0,0 +1,113 @@ +--- +# ---------------------------------------------------------------------------- +# +# *** AUTO GENERATED CODE *** AUTO GENERATED CODE *** +# +# ---------------------------------------------------------------------------- +# +# This file is automatically generated by Magic Modules and manual +# changes will be clobbered when the file is regenerated. +# +# Please read more about how to change this file at +# https://www.github.com/GoogleCloudPlatform/magic-modules +# +# ---------------------------------------------------------------------------- +# Pre-test setup +- name: delete a model + gcp_mlengine_model: + name: "{{ resource_name | replace('-', '_') }}" + description: My model + regions: + - us-central1 + project: "{{ gcp_project }}" + auth_kind: "{{ gcp_cred_kind }}" + service_account_file: "{{ gcp_cred_file }}" + state: absent +#---------------------------------------------------------- +- name: create a model + gcp_mlengine_model: + name: "{{ resource_name | replace('-', '_') }}" + description: My model + regions: + - us-central1 + project: "{{ gcp_project }}" + auth_kind: "{{ gcp_cred_kind }}" + service_account_file: "{{ gcp_cred_file }}" + state: present + register: result +- name: assert changed is true + assert: + that: + - result.changed == true +- name: verify that model was created + gcp_mlengine_model_facts: + project: "{{ gcp_project }}" + auth_kind: "{{ gcp_cred_kind }}" + service_account_file: "{{ gcp_cred_file }}" + scopes: + - https://www.googleapis.com/auth/cloud-platform + register: results +- name: verify that command succeeded + assert: + that: + - results['resources'] | map(attribute='name') | select("match", ".*{{ resource_name | replace('-', '_') }}.*") | list | length == 1 +# ---------------------------------------------------------------------------- +- name: create a model that already exists + gcp_mlengine_model: + name: "{{ resource_name | replace('-', '_') }}" + description: My model + regions: + - us-central1 + project: "{{ gcp_project }}" + auth_kind: "{{ gcp_cred_kind }}" + service_account_file: "{{ gcp_cred_file }}" + state: present + register: result +- name: assert changed is false + assert: + that: + - result.changed == false +#---------------------------------------------------------- +- name: delete a model + gcp_mlengine_model: + name: "{{ resource_name | replace('-', '_') }}" + description: My model + regions: + - us-central1 + project: "{{ gcp_project }}" + auth_kind: "{{ gcp_cred_kind }}" + service_account_file: "{{ gcp_cred_file }}" + state: absent + register: result +- name: assert changed is true + assert: + that: + - result.changed == true +- name: verify that model was deleted + gcp_mlengine_model_facts: + project: "{{ gcp_project }}" + auth_kind: "{{ gcp_cred_kind }}" + service_account_file: "{{ gcp_cred_file }}" + scopes: + - https://www.googleapis.com/auth/cloud-platform + register: results +- name: verify that command succeeded + assert: + that: + - results['resources'] | map(attribute='name') | select("match", ".*{{ resource_name | replace('-', '_') }}.*") | list | length == 0 +# ---------------------------------------------------------------------------- +- name: delete a model that does not exist + gcp_mlengine_model: + name: "{{ resource_name | replace('-', '_') }}" + description: My model + regions: + - us-central1 + project: "{{ gcp_project }}" + auth_kind: "{{ gcp_cred_kind }}" + service_account_file: "{{ gcp_cred_file }}" + state: absent + register: result +- name: assert changed is false + assert: + that: + - result.changed == false